From c7d9267375a6ef51e9b8ee1f62909b12b3bcf9ef Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Sun, 12 Oct 2025 16:48:46 +0300 Subject: [PATCH 01/15] add bitwise ops --- arrow-buffer/src/buffer/mod.rs | 2 + arrow-buffer/src/buffer/mutable.rs | 23 +- arrow-buffer/src/buffer/mutable_ops.rs | 724 ++++++++++++++++++++ arrow-buffer/src/builder/boolean.rs | 124 +++- arrow-buffer/src/util/bit_chunk_iterator.rs | 6 +- 5 files changed, 868 insertions(+), 11 deletions(-) create mode 100644 arrow-buffer/src/buffer/mutable_ops.rs diff --git a/arrow-buffer/src/buffer/mod.rs b/arrow-buffer/src/buffer/mod.rs index d33e68795e4e..676d64152e47 100644 --- a/arrow-buffer/src/buffer/mod.rs +++ b/arrow-buffer/src/buffer/mod.rs @@ -25,6 +25,8 @@ mod mutable; pub use mutable::*; mod ops; pub use ops::*; +mod mutable_ops; +pub use mutable_ops::*; mod scalar; pub use scalar::*; mod boolean; diff --git a/arrow-buffer/src/buffer/mutable.rs b/arrow-buffer/src/buffer/mutable.rs index 93d9d6b9ad84..5c52eff2e3b2 100644 --- a/arrow-buffer/src/buffer/mutable.rs +++ b/arrow-buffer/src/buffer/mutable.rs @@ -17,20 +17,18 @@ use std::alloc::{Layout, handle_alloc_error}; use std::mem; +use std::ops::AddAssign; use std::ptr::NonNull; use crate::alloc::{ALIGNMENT, Deallocation}; -use crate::{ - bytes::Bytes, - native::{ArrowNativeType, ToByteSlice}, - util::bit_util, -}; +use crate::{bitwise_unary_op_helper, bytes::Bytes, native::{ArrowNativeType, ToByteSlice}, util::bit_util}; #[cfg(feature = "pool")] use crate::pool::{MemoryPool, MemoryReservation}; #[cfg(feature = "pool")] use std::sync::Mutex; - +use crate::bit_chunk_iterator::{BitChunks, BitChunksMut, UnalignedBitChunk}; +use crate::bit_util::ceil; use super::Buffer; /// A [`MutableBuffer`] is Arrow's interface to build a [`Buffer`] out of items or slices of items. @@ -515,6 +513,19 @@ impl MutableBuffer { buffer } + /// Returns a `BitChunks` instance which can be used to iterate over this buffers bits + /// in larger chunks and starting at arbitrary bit offsets. + /// Note that both `offset` and `length` are measured in bits. + pub fn bit_chunks(&self, offset: usize, len: usize) -> BitChunks<'_> { + BitChunks::new(self.as_slice(), offset, len) + } + + /// Returns the number of 1-bits in this buffer, starting from `offset` with `length` bits + /// inspected. Note that both `offset` and `length` are measured in bits. + pub fn count_set_bits_offset(&self, offset: usize, len: usize) -> usize { + UnalignedBitChunk::new(self.as_slice(), offset, len).count_ones() + } + /// Register this [`MutableBuffer`] with the provided [`MemoryPool`] /// /// This claims the memory used by this buffer in the pool, allowing for diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs new file mode 100644 index 000000000000..319c3f63b3bd --- /dev/null +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -0,0 +1,724 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{Buffer, MutableBuffer}; +use crate::bit_chunk_iterator::BitChunks; +use crate::util::bit_util::ceil; + +fn left_mutable_bitwise_bin_op_helper( + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &[u8], + right_offset_in_bits: usize, + len_in_bits: usize, + mut op: F, +) where + F: FnMut(u64, u64) -> u64, +{ + if len_in_bits == 0 { + return; + } + + // offset inside a byte, guaranteed to be between 0 and 7 (inclusive) + let left_bit_offset = left_offset_in_bits % 8; + + let is_mutable_buffer_byte_aligned = left_bit_offset == 0; + + if is_mutable_buffer_byte_aligned { + mutable_left_byte_aligned_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right, + right_offset_in_bits, + len_in_bits, + op, + ); + } else { + // If we are not byte aligned we will read the first few bits + let bits_to_next_byte = 8 - left_bit_offset; + + align_to_byte(left, left_offset_in_bits, &right, right_offset_in_bits, &mut op, left_bit_offset, bits_to_next_byte); + + let left_offset_in_bits = left_offset_in_bits + bits_to_next_byte; + let right_offset_in_bits = right_offset_in_bits + bits_to_next_byte; + let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); + + if len_in_bits == 0 { + return; + } + + // We are now byte aligned + mutable_left_byte_aligned_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right, + right_offset_in_bits, + len_in_bits, + op, + ); + } +} + +fn align_to_byte( + op: &mut F, + buffer: &mut MutableBuffer, + offset_in_bits: usize, +) +where + F: FnMut(u64) -> u64 +{ + { + let left_bit_offset = offset_in_bits % 8; + let bits_to_next_byte = 8 - left_bit_offset; + + let left_byte_offset = offset_in_bits / 8; + let right_byte_offset = right_offset_in_bits / 8; + // 1. read the first byte from the left buffer + let left_first_byte: u8 = buffer.as_slice()[left_byte_offset]; + + // 2. Shift left byte by the left bit offset, keeping only the relevant bits + let relevant_left_first_byte = left_first_byte >> left_bit_offset; + + // 3. read the same amount of bits from the right buffer + let right_first_byte: u8 = read_up_to_byte_from_offset( + &right[right_byte_offset..], + bits_to_next_byte, + // Right bit offset + right_offset_in_bits % 8, + ); + + // 4. run the op on the first byte only + let result_first_byte = + op(relevant_left_first_byte as u64, right_first_byte as u64) as u8; + + // 5. Shift back the result to the original position + let result_first_byte = result_first_byte << left_bit_offset; + + // 6. Mask the bits that are outside the relevant bits in the left byte + // so the bits until left_bit_offset are 1 and the rest are 0 + let mask_for_first_bit_offset = (1 << left_bit_offset) - 1; + + let result_first_byte = (left_first_byte & mask_for_first_bit_offset) + | (result_first_byte & !mask_for_first_bit_offset); + + // 7. write back the result to the left buffer + buffer.as_slice_mut()[left_byte_offset] = result_first_byte; + } +} + +fn a() { + let right_byte_offset = right_offset_in_bits / 8; + + // 3. read the same amount of bits from the right buffer + let right_first_byte: u8 = read_up_to_byte_from_offset( + &right[right_byte_offset..], + bits_to_next_byte, + // Right bit offset + right_offset_in_bits % 8, + ); + + { + let left_byte_offset = left_offset_in_bits / 8; + + // 1. read the first byte from the left buffer + let left_first_byte: u8 = left.as_slice()[left_byte_offset]; + + // 2. Shift left byte by the left bit offset, keeping only the relevant bits + let relevant_left_first_byte = left_first_byte >> left_bit_offset; + + + // 4. run the op on the first byte only + let result_first_byte = + op(relevant_left_first_byte as u64, right_first_byte as u64) as u8; + + // 5. Shift back the result to the original position + let result_first_byte = result_first_byte << left_bit_offset; + + // 6. Mask the bits that are outside the relevant bits in the left byte + // so the bits until left_bit_offset are 1 and the rest are 0 + let mask_for_first_bit_offset = (1 << left_bit_offset) - 1; + + let result_first_byte = (left_first_byte & mask_for_first_bit_offset) + | (result_first_byte & !mask_for_first_bit_offset); + + // 7. write back the result to the left buffer + left.as_slice_mut()[left_byte_offset] = result_first_byte; + } +} +/// Read 8 bits from a buffer starting at a given bit offset +#[inline] +fn get_8_bits_from_offset(buffer: &Buffer, offset_in_bits: usize) -> u8 { + let byte_offset = offset_in_bits / 8; + let bit_offset = offset_in_bits % 8; + + let first_byte = buffer.as_slice()[byte_offset] as u16; + let second_byte = if byte_offset + 1 < buffer.len() { + buffer.as_slice()[byte_offset + 1] as u16 + } else { + 0 + }; + + // Combine the two bytes into a single u16 + let combined = (second_byte << 8) | first_byte; + + // Shift right by the bit offset and mask to get the relevant 8 bits + ((combined >> bit_offset) & 0xFF) as u8 +} + +#[inline] +fn read_up_to_byte_from_offset( + slice: &[u8], + number_of_bits_to_read: usize, + bit_offset: usize, +) -> u8 { + assert!(number_of_bits_to_read <= 8); + + let bit_len = number_of_bits_to_read; + if bit_len == 0 { + 0 + } else { + // number of bytes to read + // might be one more than sizeof(u64) if the offset is in the middle of a byte + let byte_len = ceil(bit_len + bit_offset, 8); + // pointer to remainder bytes after all complete chunks + let base = unsafe { slice.as_ptr() }; + + let mut bits = unsafe { std::ptr::read(base) } >> bit_offset; + for i in 1..byte_len { + let byte = unsafe { std::ptr::read(base.add(i)) }; + bits |= (byte) << (i * 8 - bit_offset); + } + + bits & ((1 << bit_len) - 1) + } +} + +/// Helper function to run the bitwise operation when we know that the left offset is byte-aligned. +/// This is the easiest case as we can do the operation directly on u64 chunks and then handle the remainder bits if any. +#[inline] +fn mutable_left_byte_aligned_bitwise_bin_op_helper( + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &[u8], + right_offset_in_bits: usize, + len_in_bits: usize, + mut op: F, +) where + F: FnMut(u64, u64) -> u64, +{ + // Must not reach here if we not byte aligned + assert_eq!( + left_offset_in_bits % 8, + 0, + "left_offset_in_bits must be byte aligned" + ); + + let right_chunks = BitChunks::new(right, right_offset_in_bits, len_in_bits); + let left_buffer_mut: &mut [u8] = { + assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len() * 8); + + let byte_offset = left_offset_in_bits / 8; + + // number of complete u64 chunks + let chunk_len = len_in_bits / 64; + + assert_eq!(right_chunks.chunk_len(), chunk_len); + + &mut left.as_slice_mut()[byte_offset..] + }; + + // cast to *const u64 should be fine since we are using read_unaligned below + #[allow(clippy::cast_ptr_alignment)] + let mut left_buffer_mut_u64_ptr = left_buffer_mut.as_mut_ptr() as *mut u64; + + let mut right_chunks_iter = right_chunks.iter(); + + // If not only remainder bytes + let had_any_chunks = right_chunks_iter.len() > 0; + + // Trying to read the first chunk + // we do this outside the loop so in the loop we can increment the pointer first + // and then read the value + // avoiding incrementing the pointer after the last read + if let Some(right) = right_chunks_iter.next() { + unsafe { + run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + } + } + + for right in right_chunks_iter { + // Increase the pointer for the next iteration + // we are increasing the pointer before reading because we already read the first chunk above + left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; + + unsafe { + run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + } + } + + // Handle remainder bits if any + if right_chunks.remainder_len() > 0 { + { + // If we had any chunks we only advance the pointer at the start. + // so we need to advance it again if we have a remainder + let advance_pointer_count = if had_any_chunks { 1 } else { 0 }; + left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(advance_pointer_count) } + } + let left_buffer_mut_u8_ptr = left_buffer_mut_u64_ptr as *mut u8; + + handle_mutable_buffer_remainder( + &mut op, + left_buffer_mut_u8_ptr, + right_chunks.remainder_bits(), + right_chunks.remainder_len(), + ) + } +} + +#[inline] +fn handle_mutable_buffer_remainder( + op: &mut F, + start_remainder_mut_ptr: *mut u8, + right_remainder_bits: u64, + remainder_len: usize, +) where + F: FnMut(u64, u64) -> u64, +{ + // Only read from mut pointer the number of remainder bits + let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); + + // Apply the operation + let rem = op(left_remainder_bits, right_remainder_bits); + + // Write only the relevant bits back the result to the mutable pointer + set_remainder_bits( + start_remainder_mut_ptr, + rem, + remainder_len, + ); +} + + +#[inline] +fn set_remainder_bits( + start_remainder_mut_ptr: *mut u8, + rem: u64, + remainder_len: usize, +) { + // Need to update the remainder bytes in the mutable buffer + // but not override the bits outside the remainder + + // Update `rem` end with the current bytes in the mutable buffer + // to preserve the bits outside the remainder + let rem = { + // 1. Read the byte that we will override + let mut current = { + let last_byte_position = remainder_len / 8; + let last_byte_ptr = unsafe { start_remainder_mut_ptr.add(last_byte_position) }; + + unsafe { std::ptr::read(last_byte_ptr) as u64 } + }; + + // Mask where the bits that are inside the remainder are 1 + // and the bits outside the remainder are 0 + let inside_remainder_mask = (1 << remainder_len) - 1; + // Mask where the bits that are outside the remainder are 1 + // and the bits inside the remainder are 0 + let outside_remainder_mask = !inside_remainder_mask; + + // 2. Only keep the bits that are outside the remainder for the value from the mutable buffer + let current = current & outside_remainder_mask; + + // 3. Only keep the bits that are inside the remainder for the value from the operation + let rem = rem & inside_remainder_mask; + + // 4. Combine the two values + current | rem + }; + + // Write back the result to the mutable pointer + { + let remainder_bytes = ceil(remainder_len, 8); + + // we are counting its starting from the least significant bit, to to_le_bytes should be correct + let rem = &rem.to_le_bytes()[0..remainder_bytes]; + + // this assumes that `[ToByteSlice]` can be copied directly + // without calling `to_byte_slice` for each element, + // which is correct for all ArrowNativeType implementations. + let src = rem.as_ptr() as *const u8; + unsafe { std::ptr::copy_nonoverlapping(src, start_remainder_mut_ptr, remainder_bytes) }; + } +} + +// Get remainder bits from a pointer and length in bits +#[inline] +fn get_remainder_bits(remainder_ptr: *const u8, remainder_len: usize) -> u64 { + let bit_len = remainder_len; + // number of bytes to read + // might be one more than sizeof(u64) if the offset is in the middle of a byte + let byte_len = ceil(bit_len, 8); + // pointer to remainder bytes after all complete chunks + let base = remainder_ptr; + + let mut bits = unsafe { std::ptr::read(base) } as u64; + for i in 1..byte_len { + let byte = unsafe { std::ptr::read(base.add(i)) }; + bits |= (byte as u64) << (i * 8); + } + + bits & ((1 << bit_len) - 1) +} + +// TODO - find a better name +#[inline] +unsafe fn run_op_on_mutable_pointer_and_single_value( + op: &mut F, + left_buffer_mut_ptr: *mut u64, + right: u64, +) where + F: FnMut(u64, u64) -> u64, +{ + // 1. Read the current value from the mutable buffer + // + // bit-packed buffers are stored starting with the least-significant byte first + // so when reading as u64 on a big-endian machine, the bytes need to be swapped + let current = unsafe { std::ptr::read_unaligned(left_buffer_mut_ptr).to_le() }; + + // 2. Get the new value by applying the operation + let combined = op(current, right); + + // 3. Write the new value back to the mutable buffer + // TODO - should write with to_le()? because we already read as to_le() + unsafe { std::ptr::write_unaligned(left_buffer_mut_ptr, combined) }; +} + + +/// Helper function to run the bitwise operation when we know that the left offset is byte-aligned. +/// This is the easiest case as we can do the operation directly on u64 chunks and then handle the remainder bits if any. +#[inline] +fn mutable_byte_aligned_bitwise_unary_op_helper( + left: &mut MutableBuffer, + left_offset_in_bits: usize, + len_in_bits: usize, + mut op: F, +) where + F: FnMut(u64) -> u64, +{ + // Must not reach here if we not byte aligned + assert_eq!( + left_offset_in_bits % 8, + 0, + "left_offset_in_bits must be byte aligned" + ); + + let number_of_u64_chunks = len_in_bits / 64; + let remainder_len = len_in_bits % 64; + + let left_buffer_mut: &mut [u8] = { + assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len() * 8); + + let byte_offset = left_offset_in_bits / 8; + + &mut left.as_slice_mut()[byte_offset..] + }; + + // cast to *const u64 should be fine since we are using read_unaligned below + #[allow(clippy::cast_ptr_alignment)] + let mut left_buffer_mut_u64_ptr = left_buffer_mut.as_mut_ptr() as *mut u64; + + // If not only remainder bytes + let had_any_chunks = number_of_u64_chunks > 0; + + // Trying to read the first chunk + // we do this outside the loop so in the loop we can increment the pointer first + // and then read the value + // avoiding incrementing the pointer after the last read + if had_any_chunks { + unsafe { + run_op_on_mutable_pointer(&mut op, left_buffer_mut_u64_ptr); + } + } + + for _ in 1..number_of_u64_chunks { + // Increase the pointer for the next iteration + // we are increasing the pointer before reading because we already read the first chunk above + left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; + + unsafe { + run_op_on_mutable_pointer(&mut op, left_buffer_mut_u64_ptr); + } + } + + // Handle remainder bits if any + if remainder_len > 0 { + { + // If we had any chunks we only advance the pointer at the start. + // so we need to advance it again if we have a remainder + let advance_pointer_count = if had_any_chunks { 1 } else { 0 }; + left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(advance_pointer_count) } + } + let left_buffer_mut_u8_ptr = left_buffer_mut_u64_ptr as *mut u8; + + handle_mutable_buffer_remainder_unary( + &mut op, + left_buffer_mut_u8_ptr, + remainder_len, + ); + } +} + +// TODO - find a better name +#[inline] +unsafe fn run_op_on_mutable_pointer( + op: &mut F, + left_buffer_mut_ptr: *mut u64, +) where + F: FnMut(u64) -> u64, +{ + // 1. Read the current value from the mutable buffer + // + // bit-packed buffers are stored starting with the least-significant byte first + // so when reading as u64 on a big-endian machine, the bytes need to be swapped + let current = unsafe { std::ptr::read_unaligned(left_buffer_mut_ptr).to_le() }; + + // 2. Get the new value by applying the operation + let result = op(current); + + // 3. Write the new value back to the mutable buffer + // TODO - should write with to_le()? because we already read as to_le() + unsafe { std::ptr::write_unaligned(left_buffer_mut_ptr, result) }; +} + +#[inline] +fn handle_mutable_buffer_remainder_unary( + op: &mut F, + start_remainder_mut_ptr: *mut u8, + remainder_len: usize, +) where + F: FnMut(u64) -> u64, +{ + // Only read from mut pointer the number of remainder bits + let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); + + // Apply the operation + let rem = op(left_remainder_bits); + + // Write only the relevant bits back the result to the mutable pointer + set_remainder_bits( + start_remainder_mut_ptr, + rem, + remainder_len, + ); +} + +/// Apply a bitwise operation `op` to the passed [`MutableBuffer`] and update it +/// The input is treated as a bitmap, meaning that offset and length are specified in number of bits. +pub fn mutable_bitwise_unary_op_helper( + left: &mut MutableBuffer, + offset_in_bits: usize, + len_in_bits: usize, + mut op: F, +) where + F: FnMut(u64) -> u64, +{ + if len_in_bits == 0 { + return; + } + + // offset inside a byte, guaranteed to be between 0 and 7 (inclusive) + let left_bit_offset = offset_in_bits % 8; + + let is_mutable_buffer_byte_aligned = left_bit_offset == 0; + + if is_mutable_buffer_byte_aligned { + mutable_byte_aligned_bitwise_unary_op_helper( + left, + offset_in_bits, + len_in_bits, + op, + ); + } else { + // If we are not byte aligned we will read the first few bits + let bits_to_next_byte = 8 - left_bit_offset; + + { + let left_byte_offset = offset_in_bits / 8; + // 1. read the first byte from the left buffer + let left_first_byte: u8 = left.as_slice()[left_byte_offset]; + + // 2. Shift left byte by the left bit offset, keeping only the relevant bits + let relevant_left_first_byte = left_first_byte >> left_bit_offset; + + // 4. run the op on the first byte only + let result_first_byte = + op(relevant_left_first_byte as u64) as u8; + + // 5. Shift back the result to the original position + let result_first_byte = result_first_byte << left_bit_offset; + + // 6. Mask the bits that are outside the relevant bits in the left byte + // so the bits until left_bit_offset are 1 and the rest are 0 + let mask_for_first_bit_offset = (1 << left_bit_offset) - 1; + + let result_first_byte = (left_first_byte & mask_for_first_bit_offset) + | (result_first_byte & !mask_for_first_bit_offset); + + // 7. write back the result to the left buffer + left.as_slice_mut()[left_byte_offset] = result_first_byte; + } + + let offset_in_bits = offset_in_bits + bits_to_next_byte; + let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); + + if len_in_bits == 0 { + return; + } + + // We are now byte aligned + mutable_byte_aligned_bitwise_unary_op_helper( + left, + offset_in_bits, + len_in_bits, + op, + ); + } +} + +/// Apply a bitwise and to two inputs and return the result as a Buffer. +/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. +pub fn left_mutable_buffer_bin_and( + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &Buffer, + right_offset_in_bits: usize, + len_in_bits: usize, +) { + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a & b, + ) +} + +/// Apply a bitwise and to two inputs and return the result as a Buffer. +/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. +pub fn both_mutable_buffer_bin_and( + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &MutableBuffer, + right_offset_in_bits: usize, + len_in_bits: usize, +) { + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a & b, + ) +} + +/// Apply a bitwise or to two inputs and return the result as a Buffer. +/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. +pub fn left_mutable_buffer_bin_or( + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &Buffer, + right_offset_in_bits: usize, + len_in_bits: usize, +) { + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a | b, + ) +} + +/// Apply a bitwise or to two inputs and return the result as a Buffer. +/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. +pub fn both_mutable_buffer_bin_or( + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &MutableBuffer, + right_offset_in_bits: usize, + len_in_bits: usize, +) { + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a | b, + ) +} + +/// Apply a bitwise xor to two inputs and return the result as a Buffer. +/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. +pub fn left_mutable_buffer_bin_xor( + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &Buffer, + right_offset_in_bits: usize, + len_in_bits: usize, +) { + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a ^ b, + ) +} + +/// Apply a bitwise xor to two inputs and return the result as a Buffer. +/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. +pub fn both_mutable_buffer_bin_xor( + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &MutableBuffer, + right_offset_in_bits: usize, + len_in_bits: usize, +) { + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a ^ b, + ) +} + +/// Apply a bitwise not to one input and return the result as a Buffer. +/// The input is treated as a bitmap, meaning that offset and length are specified in number of bits. +pub fn mutable_buffer_unary_not( + left: &mut MutableBuffer, + offset_in_bits: usize, + len_in_bits: usize, +) { + mutable_bitwise_unary_op_helper(left, offset_in_bits, len_in_bits, |a| !a) +} diff --git a/arrow-buffer/src/builder/boolean.rs b/arrow-buffer/src/builder/boolean.rs index 4ca91d1d738b..d034ffcee429 100644 --- a/arrow-buffer/src/builder/boolean.rs +++ b/arrow-buffer/src/builder/boolean.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use crate::{BooleanBuffer, Buffer, MutableBuffer, bit_mask, bit_util}; -use std::ops::Range; +use crate::{BooleanBuffer, Buffer, MutableBuffer, bit_mask, bit_util, buffer_unary_not, buffer_bin_and, buffer_bin_or, buffer_bin_xor, left_mutable_buffer_bin_and, both_mutable_buffer_bin_and, left_mutable_buffer_bin_or, both_mutable_buffer_bin_or, left_mutable_buffer_bin_xor, both_mutable_buffer_bin_xor, mutable_buffer_unary_not}; +use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not, Range}; /// Builder for [`BooleanBuffer`] /// @@ -258,6 +258,126 @@ impl BooleanBufferBuilder { } } +impl Not for BooleanBufferBuilder { + type Output = BooleanBufferBuilder; + + fn not(mut self) -> Self::Output { + mutable_buffer_unary_not(&mut self.buffer, 0, self.len); + Self { + buffer: self.buffer, + len: self.len, + } + } +} + +impl BitAnd<&BooleanBuffer> for BooleanBufferBuilder { + type Output = BooleanBufferBuilder; + + fn bitand(mut self, rhs: &BooleanBuffer) -> Self::Output { + self &= rhs; + + self + } +} + +impl BitAnd<&BooleanBufferBuilder> for BooleanBufferBuilder { + type Output = BooleanBufferBuilder; + + fn bitand(mut self, rhs: &BooleanBufferBuilder) -> Self::Output { + self &= rhs; + + self + } +} + +impl BitAndAssign<&BooleanBuffer> for BooleanBufferBuilder { + fn bitand_assign(&mut self, rhs: &BooleanBuffer) { + assert_eq!(self.len, rhs.len()); + + left_mutable_buffer_bin_and(&mut self.buffer, 0, &rhs.inner(), rhs.offset(), self.len); + } +} + +impl BitAndAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { + fn bitand_assign(&mut self, rhs: &BooleanBufferBuilder) { + assert_eq!(self.len, rhs.len()); + + both_mutable_buffer_bin_and(&mut self.buffer, 0, &rhs.buffer, 0, self.len); + } +} + +impl BitOr<&BooleanBuffer> for BooleanBufferBuilder { + type Output = BooleanBufferBuilder; + + fn bitor(mut self, rhs: &BooleanBuffer) -> Self::Output { + self |= rhs; + + self + } +} + +impl BitOr<&BooleanBufferBuilder> for BooleanBufferBuilder { + type Output = BooleanBufferBuilder; + + fn bitor(mut self, rhs: &BooleanBufferBuilder) -> Self::Output { + self |= rhs; + + self + } +} + +impl BitOrAssign<&BooleanBuffer> for BooleanBufferBuilder { + fn bitor_assign(&mut self, rhs: &BooleanBuffer) { + assert_eq!(self.len, rhs.len()); + + left_mutable_buffer_bin_or(&mut self.buffer, 0, &rhs.inner(), rhs.offset(), self.len); + } +} + +impl BitOrAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { + fn bitor_assign(&mut self, rhs: &BooleanBufferBuilder) { + assert_eq!(self.len, rhs.len()); + + both_mutable_buffer_bin_or(&mut self.buffer, 0, &rhs.buffer, 0, self.len); + } +} + +impl BitXor<&BooleanBuffer> for BooleanBufferBuilder { + type Output = BooleanBufferBuilder; + + fn bitxor(mut self, rhs: &BooleanBuffer) -> Self::Output { + self ^= rhs; + + self + } +} + +impl BitXor<&BooleanBufferBuilder> for BooleanBufferBuilder { + type Output = BooleanBufferBuilder; + + fn bitxor(mut self, rhs: &BooleanBufferBuilder) -> Self::Output { + self ^= rhs; + + self + } +} + +impl BitXorAssign<&BooleanBuffer> for BooleanBufferBuilder { + fn bitxor_assign(&mut self, rhs: &BooleanBuffer) { + assert_eq!(self.len, rhs.len()); + + left_mutable_buffer_bin_xor(&mut self.buffer, 0, &rhs.inner(), rhs.offset(), self.len); + } +} + +impl BitXorAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { + fn bitxor_assign(&mut self, rhs: &BooleanBufferBuilder) { + assert_eq!(self.len, rhs.len()); + + both_mutable_buffer_bin_xor(&mut self.buffer, 0, &rhs.buffer, 0, self.len); + } +} + impl From for Buffer { #[inline] fn from(builder: BooleanBufferBuilder) -> Self { diff --git a/arrow-buffer/src/util/bit_chunk_iterator.rs b/arrow-buffer/src/util/bit_chunk_iterator.rs index ea8e8f472ace..a4a0ede80454 100644 --- a/arrow-buffer/src/util/bit_chunk_iterator.rs +++ b/arrow-buffer/src/util/bit_chunk_iterator.rs @@ -276,8 +276,8 @@ impl<'a> BitChunks<'a> { // pointer to remainder bytes after all complete chunks let base = unsafe { self.buffer - .as_ptr() - .add(self.chunk_len * std::mem::size_of::()) + .as_ptr() + .add(self.chunk_len * std::mem::size_of::()) }; let mut bits = unsafe { std::ptr::read(base) } as u64 >> bit_offset; @@ -343,7 +343,7 @@ impl Iterator for BitChunkIterator<'_> { // the constructor ensures that bit_offset is in 0..8 // that means we need to read at most one additional byte to fill in the high bits let next = - unsafe { std::ptr::read_unaligned(raw_data.add(index + 1) as *const u8) as u64 }; + unsafe { std::ptr::read_unaligned(raw_data.add(index + 1) as *const u8) as u64 }; (current >> bit_offset) | (next << (64 - bit_offset)) }; From d14e5b7cc4a0b3da505fc9e9b5f8f29805a50a25 Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Sun, 12 Oct 2025 16:51:36 +0300 Subject: [PATCH 02/15] add bitwise ops --- arrow-buffer/src/buffer/mutable_ops.rs | 60 ++++++++------------------ 1 file changed, 18 insertions(+), 42 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index 319c3f63b3bd..2df08fb0bc68 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -51,7 +51,24 @@ fn left_mutable_bitwise_bin_op_helper( // If we are not byte aligned we will read the first few bits let bits_to_next_byte = 8 - left_bit_offset; - align_to_byte(left, left_offset_in_bits, &right, right_offset_in_bits, &mut op, left_bit_offset, bits_to_next_byte); + { + let right_byte_offset = right_offset_in_bits / 8; + + // 3. read the same amount of bits from the right buffer + let right_first_byte: u8 = read_up_to_byte_from_offset( + &right[right_byte_offset..], + bits_to_next_byte, + // Right bit offset + right_offset_in_bits % 8, + ); + + align_to_byte( + // Hope it gets inlined + &mut |left| op(left, right_first_byte as u64), + left, + left_offset_in_bits, + ); + } let left_offset_in_bits = left_offset_in_bits + bits_to_next_byte; let right_offset_in_bits = right_offset_in_bits + bits_to_next_byte; @@ -81,7 +98,6 @@ fn align_to_byte( where F: FnMut(u64) -> u64 { - { let left_bit_offset = offset_in_bits % 8; let bits_to_next_byte = 8 - left_bit_offset; @@ -117,48 +133,8 @@ where // 7. write back the result to the left buffer buffer.as_slice_mut()[left_byte_offset] = result_first_byte; - } } -fn a() { - let right_byte_offset = right_offset_in_bits / 8; - - // 3. read the same amount of bits from the right buffer - let right_first_byte: u8 = read_up_to_byte_from_offset( - &right[right_byte_offset..], - bits_to_next_byte, - // Right bit offset - right_offset_in_bits % 8, - ); - - { - let left_byte_offset = left_offset_in_bits / 8; - - // 1. read the first byte from the left buffer - let left_first_byte: u8 = left.as_slice()[left_byte_offset]; - - // 2. Shift left byte by the left bit offset, keeping only the relevant bits - let relevant_left_first_byte = left_first_byte >> left_bit_offset; - - - // 4. run the op on the first byte only - let result_first_byte = - op(relevant_left_first_byte as u64, right_first_byte as u64) as u8; - - // 5. Shift back the result to the original position - let result_first_byte = result_first_byte << left_bit_offset; - - // 6. Mask the bits that are outside the relevant bits in the left byte - // so the bits until left_bit_offset are 1 and the rest are 0 - let mask_for_first_bit_offset = (1 << left_bit_offset) - 1; - - let result_first_byte = (left_first_byte & mask_for_first_bit_offset) - | (result_first_byte & !mask_for_first_bit_offset); - - // 7. write back the result to the left buffer - left.as_slice_mut()[left_byte_offset] = result_first_byte; - } -} /// Read 8 bits from a buffer starting at a given bit offset #[inline] fn get_8_bits_from_offset(buffer: &Buffer, offset_in_bits: usize) -> u8 { From 739fe0a94a1b49178f6e3179ff08a43777814af5 Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Sun, 12 Oct 2025 16:55:51 +0300 Subject: [PATCH 03/15] cleanup --- arrow-buffer/src/buffer/mutable_ops.rs | 80 ++++++++------------------ 1 file changed, 25 insertions(+), 55 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index 2df08fb0bc68..6e0462a19d9e 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -33,7 +33,7 @@ fn left_mutable_bitwise_bin_op_helper( return; } - // offset inside a byte, guaranteed to be between 0 and 7 (inclusive) + // offset inside a byte let left_bit_offset = left_offset_in_bits % 8; let is_mutable_buffer_byte_aligned = left_bit_offset == 0; @@ -48,7 +48,7 @@ fn left_mutable_bitwise_bin_op_helper( op, ); } else { - // If we are not byte aligned we will read the first few bits + // If we are not byte aligned, run `op` on the first few bits to reach byte alignment let bits_to_next_byte = 8 - left_bit_offset; { @@ -98,41 +98,31 @@ fn align_to_byte( where F: FnMut(u64) -> u64 { - let left_bit_offset = offset_in_bits % 8; - let bits_to_next_byte = 8 - left_bit_offset; + let byte_offset = offset_in_bits / 8; + let bit_offset = offset_in_bits % 8; - let left_byte_offset = offset_in_bits / 8; - let right_byte_offset = right_offset_in_bits / 8; - // 1. read the first byte from the left buffer - let left_first_byte: u8 = buffer.as_slice()[left_byte_offset]; + // 1. read the first byte from the buffer + let first_byte: u8 = buffer.as_slice()[byte_offset]; - // 2. Shift left byte by the left bit offset, keeping only the relevant bits - let relevant_left_first_byte = left_first_byte >> left_bit_offset; - - // 3. read the same amount of bits from the right buffer - let right_first_byte: u8 = read_up_to_byte_from_offset( - &right[right_byte_offset..], - bits_to_next_byte, - // Right bit offset - right_offset_in_bits % 8, - ); + // 2. Shift byte by the bit offset, keeping only the relevant bits + let relevant_first_byte = first_byte >> bit_offset; // 4. run the op on the first byte only let result_first_byte = - op(relevant_left_first_byte as u64, right_first_byte as u64) as u8; + op(relevant_first_byte as u64) as u8; // 5. Shift back the result to the original position - let result_first_byte = result_first_byte << left_bit_offset; + let result_first_byte = result_first_byte << bit_offset; - // 6. Mask the bits that are outside the relevant bits in the left byte - // so the bits until left_bit_offset are 1 and the rest are 0 - let mask_for_first_bit_offset = (1 << left_bit_offset) - 1; + // 6. Mask the bits that are outside the relevant bits in the byte + // so the bits until bit_offset are 1 and the rest are 0 + let mask_for_first_bit_offset = (1 << bit_offset) - 1; - let result_first_byte = (left_first_byte & mask_for_first_bit_offset) + let result_first_byte = (first_byte & mask_for_first_bit_offset) | (result_first_byte & !mask_for_first_bit_offset); - // 7. write back the result to the left buffer - buffer.as_slice_mut()[left_byte_offset] = result_first_byte; + // 7. write back the result to the buffer + buffer.as_slice_mut()[byte_offset] = result_first_byte; } /// Read 8 bits from a buffer starting at a given bit offset @@ -505,7 +495,7 @@ fn handle_mutable_buffer_remainder_unary( /// Apply a bitwise operation `op` to the passed [`MutableBuffer`] and update it /// The input is treated as a bitmap, meaning that offset and length are specified in number of bits. pub fn mutable_bitwise_unary_op_helper( - left: &mut MutableBuffer, + buffer: &mut MutableBuffer, offset_in_bits: usize, len_in_bits: usize, mut op: F, @@ -516,14 +506,14 @@ pub fn mutable_bitwise_unary_op_helper( return; } - // offset inside a byte, guaranteed to be between 0 and 7 (inclusive) + // offset inside a byte let left_bit_offset = offset_in_bits % 8; let is_mutable_buffer_byte_aligned = left_bit_offset == 0; if is_mutable_buffer_byte_aligned { mutable_byte_aligned_bitwise_unary_op_helper( - left, + buffer, offset_in_bits, len_in_bits, op, @@ -532,31 +522,11 @@ pub fn mutable_bitwise_unary_op_helper( // If we are not byte aligned we will read the first few bits let bits_to_next_byte = 8 - left_bit_offset; - { - let left_byte_offset = offset_in_bits / 8; - // 1. read the first byte from the left buffer - let left_first_byte: u8 = left.as_slice()[left_byte_offset]; - - // 2. Shift left byte by the left bit offset, keeping only the relevant bits - let relevant_left_first_byte = left_first_byte >> left_bit_offset; - - // 4. run the op on the first byte only - let result_first_byte = - op(relevant_left_first_byte as u64) as u8; - - // 5. Shift back the result to the original position - let result_first_byte = result_first_byte << left_bit_offset; - - // 6. Mask the bits that are outside the relevant bits in the left byte - // so the bits until left_bit_offset are 1 and the rest are 0 - let mask_for_first_bit_offset = (1 << left_bit_offset) - 1; - - let result_first_byte = (left_first_byte & mask_for_first_bit_offset) - | (result_first_byte & !mask_for_first_bit_offset); - - // 7. write back the result to the left buffer - left.as_slice_mut()[left_byte_offset] = result_first_byte; - } + align_to_byte( + &mut op, + buffer, + offset_in_bits + ); let offset_in_bits = offset_in_bits + bits_to_next_byte; let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); @@ -567,7 +537,7 @@ pub fn mutable_bitwise_unary_op_helper( // We are now byte aligned mutable_byte_aligned_bitwise_unary_op_helper( - left, + buffer, offset_in_bits, len_in_bits, op, From 0e15b324899c07ff8d83f49ec15b08999543015d Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Sun, 12 Oct 2025 17:00:40 +0300 Subject: [PATCH 04/15] pub(crate) as I don't like that we have both mutable and only left mutable. but I don't want to pass slice of bytes as then I don't know the source and users must make sure that they hold the same promises as Buffer/MutableBuffer --- arrow-buffer/src/buffer/mutable_ops.rs | 16 ++++++++-------- arrow-buffer/src/builder/boolean.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index 6e0462a19d9e..7cced5c405c2 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -494,7 +494,7 @@ fn handle_mutable_buffer_remainder_unary( /// Apply a bitwise operation `op` to the passed [`MutableBuffer`] and update it /// The input is treated as a bitmap, meaning that offset and length are specified in number of bits. -pub fn mutable_bitwise_unary_op_helper( +pub(crate) fn mutable_bitwise_unary_op_helper( buffer: &mut MutableBuffer, offset_in_bits: usize, len_in_bits: usize, @@ -547,7 +547,7 @@ pub fn mutable_bitwise_unary_op_helper( /// Apply a bitwise and to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub fn left_mutable_buffer_bin_and( +pub(crate) fn left_mutable_buffer_bin_and( left: &mut MutableBuffer, left_offset_in_bits: usize, right: &Buffer, @@ -566,7 +566,7 @@ pub fn left_mutable_buffer_bin_and( /// Apply a bitwise and to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub fn both_mutable_buffer_bin_and( +pub(crate) fn both_mutable_buffer_bin_and( left: &mut MutableBuffer, left_offset_in_bits: usize, right: &MutableBuffer, @@ -585,7 +585,7 @@ pub fn both_mutable_buffer_bin_and( /// Apply a bitwise or to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub fn left_mutable_buffer_bin_or( +pub(crate) fn left_mutable_buffer_bin_or( left: &mut MutableBuffer, left_offset_in_bits: usize, right: &Buffer, @@ -604,7 +604,7 @@ pub fn left_mutable_buffer_bin_or( /// Apply a bitwise or to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub fn both_mutable_buffer_bin_or( +pub(crate) fn both_mutable_buffer_bin_or( left: &mut MutableBuffer, left_offset_in_bits: usize, right: &MutableBuffer, @@ -623,7 +623,7 @@ pub fn both_mutable_buffer_bin_or( /// Apply a bitwise xor to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub fn left_mutable_buffer_bin_xor( +pub(crate) fn left_mutable_buffer_bin_xor( left: &mut MutableBuffer, left_offset_in_bits: usize, right: &Buffer, @@ -642,7 +642,7 @@ pub fn left_mutable_buffer_bin_xor( /// Apply a bitwise xor to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub fn both_mutable_buffer_bin_xor( +pub(crate) fn both_mutable_buffer_bin_xor( left: &mut MutableBuffer, left_offset_in_bits: usize, right: &MutableBuffer, @@ -661,7 +661,7 @@ pub fn both_mutable_buffer_bin_xor( /// Apply a bitwise not to one input and return the result as a Buffer. /// The input is treated as a bitmap, meaning that offset and length are specified in number of bits. -pub fn mutable_buffer_unary_not( +pub(crate) fn mutable_buffer_unary_not( left: &mut MutableBuffer, offset_in_bits: usize, len_in_bits: usize, diff --git a/arrow-buffer/src/builder/boolean.rs b/arrow-buffer/src/builder/boolean.rs index d034ffcee429..cabbaf0863d0 100644 --- a/arrow-buffer/src/builder/boolean.rs +++ b/arrow-buffer/src/builder/boolean.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::{BooleanBuffer, Buffer, MutableBuffer, bit_mask, bit_util, buffer_unary_not, buffer_bin_and, buffer_bin_or, buffer_bin_xor, left_mutable_buffer_bin_and, both_mutable_buffer_bin_and, left_mutable_buffer_bin_or, both_mutable_buffer_bin_or, left_mutable_buffer_bin_xor, both_mutable_buffer_bin_xor, mutable_buffer_unary_not}; +use crate::{BooleanBuffer, Buffer, MutableBuffer, bit_mask, bit_util, left_mutable_buffer_bin_and, both_mutable_buffer_bin_and, left_mutable_buffer_bin_or, both_mutable_buffer_bin_or, left_mutable_buffer_bin_xor, both_mutable_buffer_bin_xor, mutable_buffer_unary_not}; use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not, Range}; /// Builder for [`BooleanBuffer`] From c44229992a77cdb1e28ad72f55f7f0e04e61e2ce Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Sun, 12 Oct 2025 17:38:05 +0300 Subject: [PATCH 05/15] start adding tests --- arrow-buffer/src/buffer/mutable_ops.rs | 27 +++++++++++++++++++ arrow-buffer/src/builder/boolean.rs | 37 ++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index 7cced5c405c2..63d90a89361f 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -17,8 +17,10 @@ use super::{Buffer, MutableBuffer}; use crate::bit_chunk_iterator::BitChunks; +use crate::{BooleanBuffer, BooleanBufferBuilder}; use crate::util::bit_util::ceil; + fn left_mutable_bitwise_bin_op_helper( left: &mut MutableBuffer, left_offset_in_bits: usize, @@ -668,3 +670,28 @@ pub(crate) fn mutable_buffer_unary_not( ) { mutable_bitwise_unary_op_helper(left, offset_in_bits, len_in_bits, |a| !a) } + +#[cfg(test)] +mod tests { + use std::ops::BitAnd; + use crate::{BooleanBuffer, BooleanBufferBuilder, MutableBuffer}; + + // Todo - test different types for different alignments + + fn to_boolean_vec(boolean_buffer_builder: &BooleanBufferBuilder) -> Vec { + let boolean_buffer: BooleanBuffer = boolean_buffer_builder.finish_cloned(); + boolean_buffer.iter().collect() + } + + #[test] + fn mutable_buffer_not_enough_for_single_byte() { + let input = BooleanBufferBuilder::from([ + true, false, true, false, + ].as_slice()); + + let input = !input; + // assert_eq!(to_boolean_vec(&input), ) + + + } +} diff --git a/arrow-buffer/src/builder/boolean.rs b/arrow-buffer/src/builder/boolean.rs index cabbaf0863d0..a1318b46cad8 100644 --- a/arrow-buffer/src/builder/boolean.rs +++ b/arrow-buffer/src/builder/boolean.rs @@ -256,6 +256,33 @@ impl BooleanBufferBuilder { pub fn finish_cloned(&self) -> BooleanBuffer { BooleanBuffer::new(Buffer::from_slice_ref(self.as_slice()), 0, self.len) } + + /// Returns a reference to the inner [`MutableBuffer`] + /// + /// only in tests and not public API to avoid misuse as the length of the buffer + /// is not updated when modifying the inner buffer directly + #[cfg(test)] + pub(crate) fn inner_mut(&mut self) -> &mut MutableBuffer { + &mut self.buffer + } + + /// Get the inner [`MutableBuffer`] + /// + /// Note: it might be larger than the actual length of initialized bits + /// as the bits are packed. + pub fn into_inner(self) -> MutableBuffer { + self.buffer + } + + + pub fn fixed_slice(&mut self, range: Range) -> BooleanBuffer { + assert!(range.end <= self.len); + BooleanBuffer::new( + Buffer::from_slice_ref(self.as_slice()), + range.start, + range.end - range.start, + ) + } } impl Not for BooleanBufferBuilder { @@ -392,6 +419,16 @@ impl From for BooleanBuffer { } } +impl From<&[bool]> for BooleanBufferBuilder { + #[inline] + fn from(source: &[bool]) -> Self { + let mut builder = BooleanBufferBuilder::new(source.len()); + builder.append_slice(source); + + builder + } +} + #[cfg(test)] mod tests { use super::*; From 2f28dc3f9ae5317b4608bf830f0b4647f78624f9 Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Sun, 12 Oct 2025 17:58:09 +0300 Subject: [PATCH 06/15] add tests --- arrow-buffer/src/buffer/mutable.rs | 2 +- arrow-buffer/src/buffer/mutable_ops.rs | 1249 ++++++++++++++---------- 2 files changed, 752 insertions(+), 499 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable.rs b/arrow-buffer/src/buffer/mutable.rs index 5c52eff2e3b2..3bac1e091bae 100644 --- a/arrow-buffer/src/buffer/mutable.rs +++ b/arrow-buffer/src/buffer/mutable.rs @@ -27,7 +27,7 @@ use crate::{bitwise_unary_op_helper, bytes::Bytes, native::{ArrowNativeType, ToB use crate::pool::{MemoryPool, MemoryReservation}; #[cfg(feature = "pool")] use std::sync::Mutex; -use crate::bit_chunk_iterator::{BitChunks, BitChunksMut, UnalignedBitChunk}; +use crate::bit_chunk_iterator::{BitChunks, UnalignedBitChunk}; use crate::bit_util::ceil; use super::Buffer; diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index 63d90a89361f..ebe3120a57e8 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -17,88 +17,83 @@ use super::{Buffer, MutableBuffer}; use crate::bit_chunk_iterator::BitChunks; -use crate::{BooleanBuffer, BooleanBufferBuilder}; use crate::util::bit_util::ceil; - +use crate::{BooleanBuffer, BooleanBufferBuilder}; fn left_mutable_bitwise_bin_op_helper( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &[u8], - right_offset_in_bits: usize, - len_in_bits: usize, - mut op: F, + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &[u8], + right_offset_in_bits: usize, + len_in_bits: usize, + mut op: F, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { - if len_in_bits == 0 { - return; - } - - // offset inside a byte - let left_bit_offset = left_offset_in_bits % 8; - - let is_mutable_buffer_byte_aligned = left_bit_offset == 0; - - if is_mutable_buffer_byte_aligned { - mutable_left_byte_aligned_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right, - right_offset_in_bits, - len_in_bits, - op, - ); - } else { - // If we are not byte aligned, run `op` on the first few bits to reach byte alignment - let bits_to_next_byte = 8 - left_bit_offset; - - { - let right_byte_offset = right_offset_in_bits / 8; - - // 3. read the same amount of bits from the right buffer - let right_first_byte: u8 = read_up_to_byte_from_offset( - &right[right_byte_offset..], - bits_to_next_byte, - // Right bit offset - right_offset_in_bits % 8, - ); - - align_to_byte( - // Hope it gets inlined - &mut |left| op(left, right_first_byte as u64), - left, - left_offset_in_bits, - ); - } - - let left_offset_in_bits = left_offset_in_bits + bits_to_next_byte; - let right_offset_in_bits = right_offset_in_bits + bits_to_next_byte; - let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); - if len_in_bits == 0 { - return; + return; } - // We are now byte aligned - mutable_left_byte_aligned_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right, - right_offset_in_bits, - len_in_bits, - op, - ); - } + // offset inside a byte + let left_bit_offset = left_offset_in_bits % 8; + + let is_mutable_buffer_byte_aligned = left_bit_offset == 0; + + if is_mutable_buffer_byte_aligned { + mutable_left_byte_aligned_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right, + right_offset_in_bits, + len_in_bits, + op, + ); + } else { + // If we are not byte aligned, run `op` on the first few bits to reach byte alignment + let bits_to_next_byte = 8 - left_bit_offset; + + { + let right_byte_offset = right_offset_in_bits / 8; + + // 3. read the same amount of bits from the right buffer + let right_first_byte: u8 = read_up_to_byte_from_offset( + &right[right_byte_offset..], + bits_to_next_byte, + // Right bit offset + right_offset_in_bits % 8, + ); + + align_to_byte( + // Hope it gets inlined + &mut |left| op(left, right_first_byte as u64), + left, + left_offset_in_bits, + ); + } + + let left_offset_in_bits = left_offset_in_bits + bits_to_next_byte; + let right_offset_in_bits = right_offset_in_bits + bits_to_next_byte; + let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); + + if len_in_bits == 0 { + return; + } + + // We are now byte aligned + mutable_left_byte_aligned_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right, + right_offset_in_bits, + len_in_bits, + op, + ); + } } -fn align_to_byte( - op: &mut F, - buffer: &mut MutableBuffer, - offset_in_bits: usize, -) +fn align_to_byte(op: &mut F, buffer: &mut MutableBuffer, offset_in_bits: usize) where - F: FnMut(u64) -> u64 + F: FnMut(u64) -> u64, { let byte_offset = offset_in_bits / 8; let bit_offset = offset_in_bits % 8; @@ -110,8 +105,7 @@ where let relevant_first_byte = first_byte >> bit_offset; // 4. run the op on the first byte only - let result_first_byte = - op(relevant_first_byte as u64) as u8; + let result_first_byte = op(relevant_first_byte as u64) as u8; // 5. Shift back the result to the original position let result_first_byte = result_first_byte << bit_offset; @@ -120,8 +114,8 @@ where // so the bits until bit_offset are 1 and the rest are 0 let mask_for_first_bit_offset = (1 << bit_offset) - 1; - let result_first_byte = (first_byte & mask_for_first_bit_offset) - | (result_first_byte & !mask_for_first_bit_offset); + let result_first_byte = + (first_byte & mask_for_first_bit_offset) | (result_first_byte & !mask_for_first_bit_offset); // 7. write back the result to the buffer buffer.as_slice_mut()[byte_offset] = result_first_byte; @@ -130,568 +124,827 @@ where /// Read 8 bits from a buffer starting at a given bit offset #[inline] fn get_8_bits_from_offset(buffer: &Buffer, offset_in_bits: usize) -> u8 { - let byte_offset = offset_in_bits / 8; - let bit_offset = offset_in_bits % 8; + let byte_offset = offset_in_bits / 8; + let bit_offset = offset_in_bits % 8; - let first_byte = buffer.as_slice()[byte_offset] as u16; - let second_byte = if byte_offset + 1 < buffer.len() { - buffer.as_slice()[byte_offset + 1] as u16 - } else { - 0 - }; + let first_byte = buffer.as_slice()[byte_offset] as u16; + let second_byte = if byte_offset + 1 < buffer.len() { + buffer.as_slice()[byte_offset + 1] as u16 + } else { + 0 + }; - // Combine the two bytes into a single u16 - let combined = (second_byte << 8) | first_byte; + // Combine the two bytes into a single u16 + let combined = (second_byte << 8) | first_byte; - // Shift right by the bit offset and mask to get the relevant 8 bits - ((combined >> bit_offset) & 0xFF) as u8 + // Shift right by the bit offset and mask to get the relevant 8 bits + ((combined >> bit_offset) & 0xFF) as u8 } #[inline] fn read_up_to_byte_from_offset( - slice: &[u8], - number_of_bits_to_read: usize, - bit_offset: usize, + slice: &[u8], + number_of_bits_to_read: usize, + bit_offset: usize, ) -> u8 { - assert!(number_of_bits_to_read <= 8); - - let bit_len = number_of_bits_to_read; - if bit_len == 0 { - 0 - } else { - // number of bytes to read - // might be one more than sizeof(u64) if the offset is in the middle of a byte - let byte_len = ceil(bit_len + bit_offset, 8); - // pointer to remainder bytes after all complete chunks - let base = unsafe { slice.as_ptr() }; - - let mut bits = unsafe { std::ptr::read(base) } >> bit_offset; - for i in 1..byte_len { - let byte = unsafe { std::ptr::read(base.add(i)) }; - bits |= (byte) << (i * 8 - bit_offset); + assert!(number_of_bits_to_read <= 8); + + let bit_len = number_of_bits_to_read; + if bit_len == 0 { + 0 + } else { + // number of bytes to read + // might be one more than sizeof(u64) if the offset is in the middle of a byte + let byte_len = ceil(bit_len + bit_offset, 8); + // pointer to remainder bytes after all complete chunks + let base = unsafe { slice.as_ptr() }; + + let mut bits = unsafe { std::ptr::read(base) } >> bit_offset; + for i in 1..byte_len { + let byte = unsafe { std::ptr::read(base.add(i)) }; + bits |= (byte) << (i * 8 - bit_offset); + } + + bits & ((1 << bit_len) - 1) } - - bits & ((1 << bit_len) - 1) - } } /// Helper function to run the bitwise operation when we know that the left offset is byte-aligned. /// This is the easiest case as we can do the operation directly on u64 chunks and then handle the remainder bits if any. #[inline] fn mutable_left_byte_aligned_bitwise_bin_op_helper( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &[u8], - right_offset_in_bits: usize, - len_in_bits: usize, - mut op: F, + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &[u8], + right_offset_in_bits: usize, + len_in_bits: usize, + mut op: F, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { - // Must not reach here if we not byte aligned - assert_eq!( - left_offset_in_bits % 8, - 0, - "left_offset_in_bits must be byte aligned" - ); + // Must not reach here if we not byte aligned + assert_eq!( + left_offset_in_bits % 8, + 0, + "left_offset_in_bits must be byte aligned" + ); - let right_chunks = BitChunks::new(right, right_offset_in_bits, len_in_bits); - let left_buffer_mut: &mut [u8] = { - assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len() * 8); + let right_chunks = BitChunks::new(right, right_offset_in_bits, len_in_bits); + let left_buffer_mut: &mut [u8] = { + assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len() * 8); - let byte_offset = left_offset_in_bits / 8; + let byte_offset = left_offset_in_bits / 8; - // number of complete u64 chunks - let chunk_len = len_in_bits / 64; + // number of complete u64 chunks + let chunk_len = len_in_bits / 64; - assert_eq!(right_chunks.chunk_len(), chunk_len); + assert_eq!(right_chunks.chunk_len(), chunk_len); - &mut left.as_slice_mut()[byte_offset..] - }; + &mut left.as_slice_mut()[byte_offset..] + }; - // cast to *const u64 should be fine since we are using read_unaligned below - #[allow(clippy::cast_ptr_alignment)] - let mut left_buffer_mut_u64_ptr = left_buffer_mut.as_mut_ptr() as *mut u64; + // cast to *const u64 should be fine since we are using read_unaligned below + #[allow(clippy::cast_ptr_alignment)] + let mut left_buffer_mut_u64_ptr = left_buffer_mut.as_mut_ptr() as *mut u64; - let mut right_chunks_iter = right_chunks.iter(); + let mut right_chunks_iter = right_chunks.iter(); - // If not only remainder bytes - let had_any_chunks = right_chunks_iter.len() > 0; + // If not only remainder bytes + let had_any_chunks = right_chunks_iter.len() > 0; - // Trying to read the first chunk - // we do this outside the loop so in the loop we can increment the pointer first - // and then read the value - // avoiding incrementing the pointer after the last read - if let Some(right) = right_chunks_iter.next() { - unsafe { - run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + // Trying to read the first chunk + // we do this outside the loop so in the loop we can increment the pointer first + // and then read the value + // avoiding incrementing the pointer after the last read + if let Some(right) = right_chunks_iter.next() { + unsafe { + run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + } } - } - for right in right_chunks_iter { - // Increase the pointer for the next iteration - // we are increasing the pointer before reading because we already read the first chunk above - left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; + for right in right_chunks_iter { + // Increase the pointer for the next iteration + // we are increasing the pointer before reading because we already read the first chunk above + left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; - unsafe { - run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + unsafe { + run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + } } - } - // Handle remainder bits if any - if right_chunks.remainder_len() > 0 { - { - // If we had any chunks we only advance the pointer at the start. - // so we need to advance it again if we have a remainder - let advance_pointer_count = if had_any_chunks { 1 } else { 0 }; - left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(advance_pointer_count) } - } - let left_buffer_mut_u8_ptr = left_buffer_mut_u64_ptr as *mut u8; - - handle_mutable_buffer_remainder( - &mut op, - left_buffer_mut_u8_ptr, - right_chunks.remainder_bits(), - right_chunks.remainder_len(), - ) - } + // Handle remainder bits if any + if right_chunks.remainder_len() > 0 { + { + // If we had any chunks we only advance the pointer at the start. + // so we need to advance it again if we have a remainder + let advance_pointer_count = if had_any_chunks { 1 } else { 0 }; + left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(advance_pointer_count) } + } + let left_buffer_mut_u8_ptr = left_buffer_mut_u64_ptr as *mut u8; + + handle_mutable_buffer_remainder( + &mut op, + left_buffer_mut_u8_ptr, + right_chunks.remainder_bits(), + right_chunks.remainder_len(), + ) + } } #[inline] fn handle_mutable_buffer_remainder( - op: &mut F, - start_remainder_mut_ptr: *mut u8, - right_remainder_bits: u64, - remainder_len: usize, + op: &mut F, + start_remainder_mut_ptr: *mut u8, + right_remainder_bits: u64, + remainder_len: usize, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { - // Only read from mut pointer the number of remainder bits - let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); - - // Apply the operation - let rem = op(left_remainder_bits, right_remainder_bits); - - // Write only the relevant bits back the result to the mutable pointer - set_remainder_bits( - start_remainder_mut_ptr, - rem, - remainder_len, - ); -} + // Only read from mut pointer the number of remainder bits + let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); + // Apply the operation + let rem = op(left_remainder_bits, right_remainder_bits); + + // Write only the relevant bits back the result to the mutable pointer + set_remainder_bits(start_remainder_mut_ptr, rem, remainder_len); +} #[inline] -fn set_remainder_bits( - start_remainder_mut_ptr: *mut u8, - rem: u64, - remainder_len: usize, -) { - // Need to update the remainder bytes in the mutable buffer - // but not override the bits outside the remainder - - // Update `rem` end with the current bytes in the mutable buffer - // to preserve the bits outside the remainder - let rem = { - // 1. Read the byte that we will override - let mut current = { - let last_byte_position = remainder_len / 8; - let last_byte_ptr = unsafe { start_remainder_mut_ptr.add(last_byte_position) }; - - unsafe { std::ptr::read(last_byte_ptr) as u64 } +fn set_remainder_bits(start_remainder_mut_ptr: *mut u8, rem: u64, remainder_len: usize) { + // Need to update the remainder bytes in the mutable buffer + // but not override the bits outside the remainder + + // Update `rem` end with the current bytes in the mutable buffer + // to preserve the bits outside the remainder + let rem = { + // 1. Read the byte that we will override + let mut current = { + let last_byte_position = remainder_len / 8; + let last_byte_ptr = unsafe { start_remainder_mut_ptr.add(last_byte_position) }; + + unsafe { std::ptr::read(last_byte_ptr) as u64 } + }; + + // Mask where the bits that are inside the remainder are 1 + // and the bits outside the remainder are 0 + let inside_remainder_mask = (1 << remainder_len) - 1; + // Mask where the bits that are outside the remainder are 1 + // and the bits inside the remainder are 0 + let outside_remainder_mask = !inside_remainder_mask; + + // 2. Only keep the bits that are outside the remainder for the value from the mutable buffer + let current = current & outside_remainder_mask; + + // 3. Only keep the bits that are inside the remainder for the value from the operation + let rem = rem & inside_remainder_mask; + + // 4. Combine the two values + current | rem }; - // Mask where the bits that are inside the remainder are 1 - // and the bits outside the remainder are 0 - let inside_remainder_mask = (1 << remainder_len) - 1; - // Mask where the bits that are outside the remainder are 1 - // and the bits inside the remainder are 0 - let outside_remainder_mask = !inside_remainder_mask; - - // 2. Only keep the bits that are outside the remainder for the value from the mutable buffer - let current = current & outside_remainder_mask; - - // 3. Only keep the bits that are inside the remainder for the value from the operation - let rem = rem & inside_remainder_mask; - - // 4. Combine the two values - current | rem - }; - - // Write back the result to the mutable pointer - { - let remainder_bytes = ceil(remainder_len, 8); + // Write back the result to the mutable pointer + { + let remainder_bytes = ceil(remainder_len, 8); - // we are counting its starting from the least significant bit, to to_le_bytes should be correct - let rem = &rem.to_le_bytes()[0..remainder_bytes]; + // we are counting its starting from the least significant bit, to to_le_bytes should be correct + let rem = &rem.to_le_bytes()[0..remainder_bytes]; - // this assumes that `[ToByteSlice]` can be copied directly - // without calling `to_byte_slice` for each element, - // which is correct for all ArrowNativeType implementations. - let src = rem.as_ptr() as *const u8; - unsafe { std::ptr::copy_nonoverlapping(src, start_remainder_mut_ptr, remainder_bytes) }; - } + // this assumes that `[ToByteSlice]` can be copied directly + // without calling `to_byte_slice` for each element, + // which is correct for all ArrowNativeType implementations. + let src = rem.as_ptr() as *const u8; + unsafe { std::ptr::copy_nonoverlapping(src, start_remainder_mut_ptr, remainder_bytes) }; + } } // Get remainder bits from a pointer and length in bits #[inline] fn get_remainder_bits(remainder_ptr: *const u8, remainder_len: usize) -> u64 { - let bit_len = remainder_len; - // number of bytes to read - // might be one more than sizeof(u64) if the offset is in the middle of a byte - let byte_len = ceil(bit_len, 8); - // pointer to remainder bytes after all complete chunks - let base = remainder_ptr; - - let mut bits = unsafe { std::ptr::read(base) } as u64; - for i in 1..byte_len { - let byte = unsafe { std::ptr::read(base.add(i)) }; - bits |= (byte as u64) << (i * 8); - } - - bits & ((1 << bit_len) - 1) + let bit_len = remainder_len; + // number of bytes to read + // might be one more than sizeof(u64) if the offset is in the middle of a byte + let byte_len = ceil(bit_len, 8); + // pointer to remainder bytes after all complete chunks + let base = remainder_ptr; + + let mut bits = unsafe { std::ptr::read(base) } as u64; + for i in 1..byte_len { + let byte = unsafe { std::ptr::read(base.add(i)) }; + bits |= (byte as u64) << (i * 8); + } + + bits & ((1 << bit_len) - 1) } // TODO - find a better name #[inline] unsafe fn run_op_on_mutable_pointer_and_single_value( - op: &mut F, - left_buffer_mut_ptr: *mut u64, - right: u64, + op: &mut F, + left_buffer_mut_ptr: *mut u64, + right: u64, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { - // 1. Read the current value from the mutable buffer - // - // bit-packed buffers are stored starting with the least-significant byte first - // so when reading as u64 on a big-endian machine, the bytes need to be swapped - let current = unsafe { std::ptr::read_unaligned(left_buffer_mut_ptr).to_le() }; - - // 2. Get the new value by applying the operation - let combined = op(current, right); - - // 3. Write the new value back to the mutable buffer - // TODO - should write with to_le()? because we already read as to_le() - unsafe { std::ptr::write_unaligned(left_buffer_mut_ptr, combined) }; + // 1. Read the current value from the mutable buffer + // + // bit-packed buffers are stored starting with the least-significant byte first + // so when reading as u64 on a big-endian machine, the bytes need to be swapped + let current = unsafe { std::ptr::read_unaligned(left_buffer_mut_ptr).to_le() }; + + // 2. Get the new value by applying the operation + let combined = op(current, right); + + // 3. Write the new value back to the mutable buffer + // TODO - should write with to_le()? because we already read as to_le() + unsafe { std::ptr::write_unaligned(left_buffer_mut_ptr, combined) }; } - /// Helper function to run the bitwise operation when we know that the left offset is byte-aligned. /// This is the easiest case as we can do the operation directly on u64 chunks and then handle the remainder bits if any. #[inline] fn mutable_byte_aligned_bitwise_unary_op_helper( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - len_in_bits: usize, - mut op: F, + left: &mut MutableBuffer, + left_offset_in_bits: usize, + len_in_bits: usize, + mut op: F, ) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { - // Must not reach here if we not byte aligned - assert_eq!( - left_offset_in_bits % 8, - 0, - "left_offset_in_bits must be byte aligned" - ); - - let number_of_u64_chunks = len_in_bits / 64; - let remainder_len = len_in_bits % 64; - - let left_buffer_mut: &mut [u8] = { - assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len() * 8); - - let byte_offset = left_offset_in_bits / 8; - - &mut left.as_slice_mut()[byte_offset..] - }; - - // cast to *const u64 should be fine since we are using read_unaligned below - #[allow(clippy::cast_ptr_alignment)] - let mut left_buffer_mut_u64_ptr = left_buffer_mut.as_mut_ptr() as *mut u64; - - // If not only remainder bytes - let had_any_chunks = number_of_u64_chunks > 0; - - // Trying to read the first chunk - // we do this outside the loop so in the loop we can increment the pointer first - // and then read the value - // avoiding incrementing the pointer after the last read - if had_any_chunks { - unsafe { - run_op_on_mutable_pointer(&mut op, left_buffer_mut_u64_ptr); - } - } - - for _ in 1..number_of_u64_chunks { - // Increase the pointer for the next iteration - // we are increasing the pointer before reading because we already read the first chunk above - left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; - - unsafe { - run_op_on_mutable_pointer(&mut op, left_buffer_mut_u64_ptr); - } - } - - // Handle remainder bits if any - if remainder_len > 0 { - { - // If we had any chunks we only advance the pointer at the start. - // so we need to advance it again if we have a remainder - let advance_pointer_count = if had_any_chunks { 1 } else { 0 }; - left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(advance_pointer_count) } + // Must not reach here if we not byte aligned + assert_eq!( + left_offset_in_bits % 8, + 0, + "left_offset_in_bits must be byte aligned" + ); + + let number_of_u64_chunks = len_in_bits / 64; + let remainder_len = len_in_bits % 64; + + let left_buffer_mut: &mut [u8] = { + assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len() * 8); + + let byte_offset = left_offset_in_bits / 8; + + &mut left.as_slice_mut()[byte_offset..] + }; + + // cast to *const u64 should be fine since we are using read_unaligned below + #[allow(clippy::cast_ptr_alignment)] + let mut left_buffer_mut_u64_ptr = left_buffer_mut.as_mut_ptr() as *mut u64; + + // If not only remainder bytes + let had_any_chunks = number_of_u64_chunks > 0; + + // Trying to read the first chunk + // we do this outside the loop so in the loop we can increment the pointer first + // and then read the value + // avoiding incrementing the pointer after the last read + if had_any_chunks { + unsafe { + run_op_on_mutable_pointer(&mut op, left_buffer_mut_u64_ptr); + } } - let left_buffer_mut_u8_ptr = left_buffer_mut_u64_ptr as *mut u8; - handle_mutable_buffer_remainder_unary( - &mut op, - left_buffer_mut_u8_ptr, - remainder_len, - ); - } + for _ in 1..number_of_u64_chunks { + // Increase the pointer for the next iteration + // we are increasing the pointer before reading because we already read the first chunk above + left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; + + unsafe { + run_op_on_mutable_pointer(&mut op, left_buffer_mut_u64_ptr); + } + } + + // Handle remainder bits if any + if remainder_len > 0 { + { + // If we had any chunks we only advance the pointer at the start. + // so we need to advance it again if we have a remainder + let advance_pointer_count = if had_any_chunks { 1 } else { 0 }; + left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(advance_pointer_count) } + } + let left_buffer_mut_u8_ptr = left_buffer_mut_u64_ptr as *mut u8; + + handle_mutable_buffer_remainder_unary(&mut op, left_buffer_mut_u8_ptr, remainder_len); + } } // TODO - find a better name #[inline] -unsafe fn run_op_on_mutable_pointer( - op: &mut F, - left_buffer_mut_ptr: *mut u64, -) where - F: FnMut(u64) -> u64, +unsafe fn run_op_on_mutable_pointer(op: &mut F, left_buffer_mut_ptr: *mut u64) +where + F: FnMut(u64) -> u64, { - // 1. Read the current value from the mutable buffer - // - // bit-packed buffers are stored starting with the least-significant byte first - // so when reading as u64 on a big-endian machine, the bytes need to be swapped - let current = unsafe { std::ptr::read_unaligned(left_buffer_mut_ptr).to_le() }; - - // 2. Get the new value by applying the operation - let result = op(current); - - // 3. Write the new value back to the mutable buffer - // TODO - should write with to_le()? because we already read as to_le() - unsafe { std::ptr::write_unaligned(left_buffer_mut_ptr, result) }; + // 1. Read the current value from the mutable buffer + // + // bit-packed buffers are stored starting with the least-significant byte first + // so when reading as u64 on a big-endian machine, the bytes need to be swapped + let current = unsafe { std::ptr::read_unaligned(left_buffer_mut_ptr).to_le() }; + + // 2. Get the new value by applying the operation + let result = op(current); + + // 3. Write the new value back to the mutable buffer + // TODO - should write with to_le()? because we already read as to_le() + unsafe { std::ptr::write_unaligned(left_buffer_mut_ptr, result) }; } #[inline] fn handle_mutable_buffer_remainder_unary( - op: &mut F, - start_remainder_mut_ptr: *mut u8, - remainder_len: usize, + op: &mut F, + start_remainder_mut_ptr: *mut u8, + remainder_len: usize, ) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { - // Only read from mut pointer the number of remainder bits - let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); - - // Apply the operation - let rem = op(left_remainder_bits); - - // Write only the relevant bits back the result to the mutable pointer - set_remainder_bits( - start_remainder_mut_ptr, - rem, - remainder_len, - ); + // Only read from mut pointer the number of remainder bits + let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); + + // Apply the operation + let rem = op(left_remainder_bits); + + // Write only the relevant bits back the result to the mutable pointer + set_remainder_bits(start_remainder_mut_ptr, rem, remainder_len); } /// Apply a bitwise operation `op` to the passed [`MutableBuffer`] and update it /// The input is treated as a bitmap, meaning that offset and length are specified in number of bits. pub(crate) fn mutable_bitwise_unary_op_helper( - buffer: &mut MutableBuffer, - offset_in_bits: usize, - len_in_bits: usize, - mut op: F, + buffer: &mut MutableBuffer, + offset_in_bits: usize, + len_in_bits: usize, + mut op: F, ) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { - if len_in_bits == 0 { - return; - } + if len_in_bits == 0 { + return; + } - // offset inside a byte - let left_bit_offset = offset_in_bits % 8; + // offset inside a byte + let left_bit_offset = offset_in_bits % 8; - let is_mutable_buffer_byte_aligned = left_bit_offset == 0; + let is_mutable_buffer_byte_aligned = left_bit_offset == 0; - if is_mutable_buffer_byte_aligned { - mutable_byte_aligned_bitwise_unary_op_helper( - buffer, - offset_in_bits, - len_in_bits, - op, - ); - } else { - // If we are not byte aligned we will read the first few bits - let bits_to_next_byte = 8 - left_bit_offset; - - align_to_byte( - &mut op, - buffer, - offset_in_bits - ); + if is_mutable_buffer_byte_aligned { + mutable_byte_aligned_bitwise_unary_op_helper(buffer, offset_in_bits, len_in_bits, op); + } else { + // If we are not byte aligned we will read the first few bits + let bits_to_next_byte = 8 - left_bit_offset; - let offset_in_bits = offset_in_bits + bits_to_next_byte; - let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); + align_to_byte(&mut op, buffer, offset_in_bits); - if len_in_bits == 0 { - return; - } + let offset_in_bits = offset_in_bits + bits_to_next_byte; + let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); - // We are now byte aligned - mutable_byte_aligned_bitwise_unary_op_helper( - buffer, - offset_in_bits, - len_in_bits, - op, - ); - } + if len_in_bits == 0 { + return; + } + + // We are now byte aligned + mutable_byte_aligned_bitwise_unary_op_helper(buffer, offset_in_bits, len_in_bits, op); + } } /// Apply a bitwise and to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. pub(crate) fn left_mutable_buffer_bin_and( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &Buffer, - right_offset_in_bits: usize, - len_in_bits: usize, + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &Buffer, + right_offset_in_bits: usize, + len_in_bits: usize, ) { - left_mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right.as_slice(), - right_offset_in_bits, - len_in_bits, - |a, b| a & b, - ) + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a & b, + ) } /// Apply a bitwise and to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. pub(crate) fn both_mutable_buffer_bin_and( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &MutableBuffer, - right_offset_in_bits: usize, - len_in_bits: usize, + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &MutableBuffer, + right_offset_in_bits: usize, + len_in_bits: usize, ) { - left_mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right.as_slice(), - right_offset_in_bits, - len_in_bits, - |a, b| a & b, - ) + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a & b, + ) } /// Apply a bitwise or to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. pub(crate) fn left_mutable_buffer_bin_or( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &Buffer, - right_offset_in_bits: usize, - len_in_bits: usize, + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &Buffer, + right_offset_in_bits: usize, + len_in_bits: usize, ) { - left_mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right.as_slice(), - right_offset_in_bits, - len_in_bits, - |a, b| a | b, - ) + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a | b, + ) } /// Apply a bitwise or to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. pub(crate) fn both_mutable_buffer_bin_or( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &MutableBuffer, - right_offset_in_bits: usize, - len_in_bits: usize, + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &MutableBuffer, + right_offset_in_bits: usize, + len_in_bits: usize, ) { - left_mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right.as_slice(), - right_offset_in_bits, - len_in_bits, - |a, b| a | b, - ) + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a | b, + ) } /// Apply a bitwise xor to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. pub(crate) fn left_mutable_buffer_bin_xor( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &Buffer, - right_offset_in_bits: usize, - len_in_bits: usize, + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &Buffer, + right_offset_in_bits: usize, + len_in_bits: usize, ) { - left_mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right.as_slice(), - right_offset_in_bits, - len_in_bits, - |a, b| a ^ b, - ) + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a ^ b, + ) } /// Apply a bitwise xor to two inputs and return the result as a Buffer. /// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. pub(crate) fn both_mutable_buffer_bin_xor( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &MutableBuffer, - right_offset_in_bits: usize, - len_in_bits: usize, + left: &mut MutableBuffer, + left_offset_in_bits: usize, + right: &MutableBuffer, + right_offset_in_bits: usize, + len_in_bits: usize, ) { - left_mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right.as_slice(), - right_offset_in_bits, - len_in_bits, - |a, b| a ^ b, - ) + left_mutable_bitwise_bin_op_helper( + left, + left_offset_in_bits, + right.as_slice(), + right_offset_in_bits, + len_in_bits, + |a, b| a ^ b, + ) } /// Apply a bitwise not to one input and return the result as a Buffer. /// The input is treated as a bitmap, meaning that offset and length are specified in number of bits. pub(crate) fn mutable_buffer_unary_not( - left: &mut MutableBuffer, - offset_in_bits: usize, - len_in_bits: usize, + left: &mut MutableBuffer, + offset_in_bits: usize, + len_in_bits: usize, ) { - mutable_bitwise_unary_op_helper(left, offset_in_bits, len_in_bits, |a| !a) + mutable_bitwise_unary_op_helper(left, offset_in_bits, len_in_bits, |a| !a) } - #[cfg(test)] mod tests { - use std::ops::BitAnd; - use crate::{BooleanBuffer, BooleanBufferBuilder, MutableBuffer}; + use crate::bit_iterator::BitIterator; + use crate::{BooleanBuffer, BooleanBufferBuilder}; + + fn test_mutable_buffer_bin_op_helper( + left_data: &[bool], + right_data: &[bool], + left_offset_in_bits: usize, + right_offset_in_bits: usize, + len_in_bits: usize, + op: F, + mut expected_op: G, + ) where + F: FnMut(u64, u64) -> u64, + G: FnMut(bool, bool) -> bool, + { + let mut left_buffer = BooleanBufferBuilder::from(left_data).into_inner(); + let right_buffer = BooleanBuffer::from(right_data); + + let expected: Vec = left_data + .iter() + .skip(left_offset_in_bits) + .zip(right_data.iter().skip(right_offset_in_bits)) + .take(len_in_bits) + .map(|(l, r)| expected_op(*l, *r)) + .collect(); + + super::left_mutable_bitwise_bin_op_helper( + &mut left_buffer, + left_offset_in_bits, + right_buffer.values(), + right_offset_in_bits, + len_in_bits, + op, + ); + + let result: Vec = + BitIterator::new(left_buffer.as_slice(), left_offset_in_bits, len_in_bits).collect(); + + assert_eq!( + result, expected, + "Failed with left_offset={}, right_offset={}, len={}", + left_offset_in_bits, right_offset_in_bits, len_in_bits + ); + } + + fn test_mutable_buffer_unary_op_helper( + data: &[bool], + offset_in_bits: usize, + len_in_bits: usize, + op: F, + mut expected_op: G, + ) where + F: FnMut(u64) -> u64, + G: FnMut(bool) -> bool, + { + let mut buffer = BooleanBufferBuilder::from(data).into_inner(); + + let expected: Vec = data + .iter() + .skip(offset_in_bits) + .take(len_in_bits) + .map(|b| expected_op(*b)) + .collect(); + + super::mutable_bitwise_unary_op_helper(&mut buffer, offset_in_bits, len_in_bits, op); + + let result: Vec = + BitIterator::new(buffer.as_slice(), offset_in_bits, len_in_bits).collect(); + + assert_eq!( + result, expected, + "Failed with offset={}, len={}", + offset_in_bits, len_in_bits + ); + } + + // Helper to create test data of specific length + fn create_test_data(len: usize) -> (Vec, Vec) { + let left: Vec = (0..len).map(|i| i % 2 == 0).collect(); + let right: Vec = (0..len).map(|i| (i / 2) % 2 == 0).collect(); + (left, right) + } + + /// Test all binary operations (AND, OR, XOR) with the given parameters + fn test_all_binary_ops( + left_data: &[bool], + right_data: &[bool], + left_offset_in_bits: usize, + right_offset_in_bits: usize, + len_in_bits: usize, + ) { + // Test AND + test_mutable_buffer_bin_op_helper( + left_data, + right_data, + left_offset_in_bits, + right_offset_in_bits, + len_in_bits, + |a, b| a & b, + |a, b| a & b, + ); + + // Test OR + test_mutable_buffer_bin_op_helper( + left_data, + right_data, + left_offset_in_bits, + right_offset_in_bits, + len_in_bits, + |a, b| a | b, + |a, b| a | b, + ); + + // Test XOR + test_mutable_buffer_bin_op_helper( + left_data, + right_data, + left_offset_in_bits, + right_offset_in_bits, + len_in_bits, + |a, b| a ^ b, + |a, b| a ^ b, + ); + } + + // ===== Combined Binary Operation Tests ===== + + #[test] + fn test_binary_ops_less_than_byte() { + let (left, right) = create_test_data(4); + test_all_binary_ops(&left, &right, 0, 0, 4); + } + + #[test] + fn test_binary_ops_less_than_byte_across_boundary() { + let (left, right) = create_test_data(16); + test_all_binary_ops(&left, &right, 6, 6, 4); + } + + #[test] + fn test_binary_ops_exactly_byte() { + let (left, right) = create_test_data(16); + test_all_binary_ops(&left, &right, 0, 0, 8); + } + + #[test] + fn test_binary_ops_more_than_byte_less_than_u64() { + let (left, right) = create_test_data(64); + test_all_binary_ops(&left, &right, 0, 0, 32); + } + + #[test] + fn test_binary_ops_exactly_u64() { + let (left, right) = create_test_data(180); + test_all_binary_ops(&left, &right, 0, 0, 64); + test_all_binary_ops(&left, &right, 64, 9, 64); + test_all_binary_ops(&left, &right, 8, 100, 64); + test_all_binary_ops(&left, &right, 1, 15, 64); + test_all_binary_ops(&left, &right, 12, 10, 64); + test_all_binary_ops(&left, &right, 180 - 64, 2, 64); + } + + #[test] + fn test_binary_ops_more_than_u64_not_multiple() { + let (left, right) = create_test_data(200); + test_all_binary_ops(&left, &right, 0, 0, 100); + } + + #[test] + fn test_binary_ops_exactly_multiple_u64() { + let (left, right) = create_test_data(256); + test_all_binary_ops(&left, &right, 0, 0, 128); + } + + #[test] + fn test_binary_ops_more_than_multiple_u64() { + let (left, right) = create_test_data(300); + test_all_binary_ops(&left, &right, 0, 0, 200); + } - // Todo - test different types for different alignments + #[test] + fn test_binary_ops_byte_aligned_no_remainder() { + let (left, right) = create_test_data(200); + test_all_binary_ops(&left, &right, 0, 0, 128); + } - fn to_boolean_vec(boolean_buffer_builder: &BooleanBufferBuilder) -> Vec { - let boolean_buffer: BooleanBuffer = boolean_buffer_builder.finish_cloned(); - boolean_buffer.iter().collect() - } + #[test] + fn test_binary_ops_byte_aligned_with_remainder() { + let (left, right) = create_test_data(200); + test_all_binary_ops(&left, &right, 0, 0, 100); + } - #[test] - fn mutable_buffer_not_enough_for_single_byte() { - let input = BooleanBufferBuilder::from([ - true, false, true, false, - ].as_slice()); + #[test] + fn test_binary_ops_not_byte_aligned_no_remainder() { + let (left, right) = create_test_data(200); + test_all_binary_ops(&left, &right, 3, 3, 128); + } - let input = !input; - // assert_eq!(to_boolean_vec(&input), ) + #[test] + fn test_binary_ops_not_byte_aligned_with_remainder() { + let (left, right) = create_test_data(200); + test_all_binary_ops(&left, &right, 5, 5, 100); + } + #[test] + fn test_binary_ops_different_offsets() { + let (left, right) = create_test_data(200); + test_all_binary_ops(&left, &right, 3, 7, 50); + } + + // ===== NOT (Unary) Operation Tests ===== + + #[test] + fn test_not_less_than_byte() { + let data = vec![true, false, true, false]; + test_mutable_buffer_unary_op_helper(&data, 0, 4, |a| !a, |a| !a); + } + + #[test] + fn test_not_less_than_byte_across_boundary() { + let data: Vec = (0..16).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 6, 4, |a| !a, |a| !a); + } + + #[test] + fn test_not_exactly_byte() { + let data: Vec = (0..16).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 0, 8, |a| !a, |a| !a); + } + + #[test] + fn test_not_more_than_byte_less_than_u64() { + let data: Vec = (0..64).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 0, 32, |a| !a, |a| !a); + } + + #[test] + fn test_not_exactly_u64() { + let data: Vec = (0..128).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 0, 64, |a| !a, |a| !a); + } + + #[test] + fn test_not_more_than_u64_not_multiple() { + let data: Vec = (0..200).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 0, 100, |a| !a, |a| !a); + } + + #[test] + fn test_not_exactly_multiple_u64() { + let data: Vec = (0..256).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 0, 128, |a| !a, |a| !a); + } - } + #[test] + fn test_not_more_than_multiple_u64() { + let data: Vec = (0..300).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 0, 200, |a| !a, |a| !a); + } + + #[test] + fn test_not_byte_aligned_no_remainder() { + let data: Vec = (0..200).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 0, 128, |a| !a, |a| !a); + } + + #[test] + fn test_not_byte_aligned_with_remainder() { + let data: Vec = (0..200).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 0, 100, |a| !a, |a| !a); + } + + #[test] + fn test_not_not_byte_aligned_no_remainder() { + let data: Vec = (0..200).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 3, 128, |a| !a, |a| !a); + } + + #[test] + fn test_not_not_byte_aligned_with_remainder() { + let data: Vec = (0..200).map(|i| i % 2 == 0).collect(); + test_mutable_buffer_unary_op_helper(&data, 5, 100, |a| !a, |a| !a); + } + + // ===== Edge Cases ===== + + #[test] + fn test_empty_length() { + let (left, right) = create_test_data(16); + test_all_binary_ops(&left, &right, 0, 0, 0); + } + + #[test] + fn test_single_bit() { + let (left, right) = create_test_data(16); + test_all_binary_ops(&left, &right, 0, 0, 1); + } + + #[test] + fn test_single_bit_at_offset() { + let (left, right) = create_test_data(16); + test_all_binary_ops(&left, &right, 7, 7, 1); + } + + #[test] + fn test_not_single_bit() { + let data = vec![true, false, true, false]; + test_mutable_buffer_unary_op_helper(&data, 0, 1, |a| !a, |a| !a); + } + + #[test] + fn test_not_empty_length() { + let data = vec![true, false, true, false]; + test_mutable_buffer_unary_op_helper(&data, 0, 0, |a| !a, |a| !a); + } } From c4676a60375e716f4cfe9203003176c3ffded28a Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Wed, 15 Oct 2025 10:29:27 +0300 Subject: [PATCH 07/15] add trait for left --- arrow-buffer/src/buffer/mutable.rs | 4 +- arrow-buffer/src/buffer/mutable_ops.rs | 576 +++++++++++++++++-------- arrow-buffer/src/builder/boolean.rs | 42 +- 3 files changed, 410 insertions(+), 212 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable.rs b/arrow-buffer/src/buffer/mutable.rs index 3bac1e091bae..bdfd790deec0 100644 --- a/arrow-buffer/src/buffer/mutable.rs +++ b/arrow-buffer/src/buffer/mutable.rs @@ -17,18 +17,16 @@ use std::alloc::{Layout, handle_alloc_error}; use std::mem; -use std::ops::AddAssign; use std::ptr::NonNull; use crate::alloc::{ALIGNMENT, Deallocation}; -use crate::{bitwise_unary_op_helper, bytes::Bytes, native::{ArrowNativeType, ToByteSlice}, util::bit_util}; +use crate::{bytes::Bytes, native::{ArrowNativeType, ToByteSlice}, util::bit_util}; #[cfg(feature = "pool")] use crate::pool::{MemoryPool, MemoryReservation}; #[cfg(feature = "pool")] use std::sync::Mutex; use crate::bit_chunk_iterator::{BitChunks, UnalignedBitChunk}; -use crate::bit_util::ceil; use super::Buffer; /// A [`MutableBuffer`] is Arrow's interface to build a [`Buffer`] out of items or slices of items. diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index ebe3120a57e8..8887272b3079 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -17,31 +17,122 @@ use super::{Buffer, MutableBuffer}; use crate::bit_chunk_iterator::BitChunks; +use crate::BooleanBufferBuilder; use crate::util::bit_util::ceil; -use crate::{BooleanBuffer, BooleanBufferBuilder}; -fn left_mutable_bitwise_bin_op_helper( - left: &mut MutableBuffer, + +/// What can be used as the right-hand side (RHS) buffer in mutable operations. +/// +/// this is not mutated. +/// +/// # Implementation notes +/// +/// ## Why `pub(crate)`? +/// This is because we don't want this trait to expose the inner buffer to the public. +/// this is the trait implementor choice. +/// +pub(crate) trait BufferSupportedRhs { + fn as_slice(&self) -> &[u8]; +} + +impl BufferSupportedRhs for Buffer { + fn as_slice(&self) -> &[u8] { + self.as_slice() + } +} + +impl BufferSupportedRhs for MutableBuffer { + fn as_slice(&self) -> &[u8] { + self.as_slice() + } +} + +impl BufferSupportedRhs for BooleanBufferBuilder { + fn as_slice(&self) -> &[u8] { + self.as_slice() + } +} + +/// Trait that will be operated on as the left-hand side (LHS) buffer in mutable operations. +/// +/// This consumer of the trait must satisfies the following guarantees: +/// 1. It will not change the length of the buffer. +/// +/// # Implementation notes +/// +/// ## Why is this trait `pub(crate)`? +/// Because we don't wanna expose the inner mutable buffer to the public. +/// as this is the choice of the implementor of the trait and sometimes it is not desirable +/// (e.g. `BooleanBufferBuilder`). +/// +/// ## Why this trait is needed, can't we just use `MutableBuffer` directly? +/// Sometimes we don't want to expose the inner `MutableBuffer` +/// so it can't be misused. +/// +/// For example, [`BooleanBufferBuilder`] does not expose the inner `MutableBuffer` +/// as exposing it will allow the user to change the length of the buffer that will make the +/// `BooleanBufferBuilder` invalid. +/// +pub(crate) trait MutableOpsBufferSupportedLhs { + /// Get a mutable reference to the inner `MutableBuffer`. + /// + /// This is used to perform in-place operations on the buffer. + /// + /// the caller must ensure that the length of the buffer is not changed. + fn inner_mutable_buffer(&mut self) -> &mut MutableBuffer; +} + +impl MutableOpsBufferSupportedLhs for MutableBuffer { + fn inner_mutable_buffer(&mut self) -> &mut MutableBuffer { + self + } +} + +/// Apply a binary bitwise operation to two bit-packed buffers. +/// +/// This is the main entry point for binary operations. It handles both byte-aligned +/// and non-byte-aligned cases by delegating to specialized helper functions. +/// +/// # Arguments +/// +/// * `left` - The left mutable buffer to be modified in-place +/// * `left_offset_in_bits` - Starting bit offset in the left buffer +/// * `right` - The right buffer (as byte slice) +/// * `right_offset_in_bits` - Starting bit offset in the right buffer +/// * `len_in_bits` - Number of bits to process +/// * `op` - Binary operation to apply (e.g., `|a, b| a & b`) +/// +#[allow( + private_bounds, + reason = "MutableOpsBufferSupportedLhs and BufferSupportedRhs exposes the inner internals which is the implementor choice and we dont want to leak internals" +)] +pub fn mutable_bitwise_bin_op_helper( + left: &mut impl MutableOpsBufferSupportedLhs, left_offset_in_bits: usize, - right: &[u8], + right: &impl BufferSupportedRhs, right_offset_in_bits: usize, len_in_bits: usize, mut op: F, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { if len_in_bits == 0 { return; } + let mutable_buffer = left.inner_mutable_buffer(); + + let mutable_buffer_len = mutable_buffer.len(); + let mutable_buffer_cap = mutable_buffer.capacity(); + // offset inside a byte let left_bit_offset = left_offset_in_bits % 8; let is_mutable_buffer_byte_aligned = left_bit_offset == 0; if is_mutable_buffer_byte_aligned { - mutable_left_byte_aligned_bitwise_bin_op_helper( - left, + mutable_buffer_byte_aligned_bitwise_bin_op_helper( + mutable_buffer, left_offset_in_bits, right, right_offset_in_bits, @@ -55,9 +146,9 @@ fn left_mutable_bitwise_bin_op_helper( { let right_byte_offset = right_offset_in_bits / 8; - // 3. read the same amount of bits from the right buffer + // Read the same amount of bits from the right buffer let right_first_byte: u8 = read_up_to_byte_from_offset( - &right[right_byte_offset..], + &right.as_slice()[right_byte_offset..], bits_to_next_byte, // Right bit offset right_offset_in_bits % 8, @@ -66,7 +157,7 @@ fn left_mutable_bitwise_bin_op_helper( align_to_byte( // Hope it gets inlined &mut |left| op(left, right_first_byte as u64), - left, + mutable_buffer, left_offset_in_bits, ); } @@ -76,12 +167,17 @@ fn left_mutable_bitwise_bin_op_helper( let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); if len_in_bits == 0 { + // Making sure that our guarantee that the length and capacity of the mutable buffer + // will not change is upheld + assert_eq!(mutable_buffer.len(), mutable_buffer_len, "The length of the mutable buffer must not change"); + assert_eq!(mutable_buffer.capacity(), mutable_buffer_cap, "The capacity of the mutable buffer must not change"); + return; } // We are now byte aligned - mutable_left_byte_aligned_bitwise_bin_op_helper( - left, + mutable_buffer_byte_aligned_bitwise_bin_op_helper( + mutable_buffer, left_offset_in_bits, right, right_offset_in_bits, @@ -89,11 +185,26 @@ fn left_mutable_bitwise_bin_op_helper( op, ); } + + // Making sure that our guarantee that the length and capacity of the mutable buffer + // will not change is upheld + assert_eq!(mutable_buffer.len(), mutable_buffer_len, "The length of the mutable buffer must not change"); + assert_eq!(mutable_buffer.capacity(), mutable_buffer_cap, "The capacity of the mutable buffer must not change"); } +/// Align to byte boundary by applying operation to bits before the next byte boundary. +/// +/// This function handles non-byte-aligned operations by processing bits from the current +/// position up to the next byte boundary, while preserving all other bits in the byte. +/// +/// # Arguments +/// +/// * `op` - Unary operation to apply +/// * `buffer` - The mutable buffer to modify +/// * `offset_in_bits` - Starting bit offset (not byte-aligned) fn align_to_byte(op: &mut F, buffer: &mut MutableBuffer, offset_in_bits: usize) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { let byte_offset = offset_in_bits / 8; let bit_offset = offset_in_bits % 8; @@ -104,43 +215,37 @@ where // 2. Shift byte by the bit offset, keeping only the relevant bits let relevant_first_byte = first_byte >> bit_offset; - // 4. run the op on the first byte only + // 3. run the op on the first byte only let result_first_byte = op(relevant_first_byte as u64) as u8; - // 5. Shift back the result to the original position + // 4. Shift back the result to the original position let result_first_byte = result_first_byte << bit_offset; - // 6. Mask the bits that are outside the relevant bits in the byte + // 5. Mask the bits that are outside the relevant bits in the byte // so the bits until bit_offset are 1 and the rest are 0 let mask_for_first_bit_offset = (1 << bit_offset) - 1; let result_first_byte = - (first_byte & mask_for_first_bit_offset) | (result_first_byte & !mask_for_first_bit_offset); + (first_byte & mask_for_first_bit_offset) | (result_first_byte & !mask_for_first_bit_offset); - // 7. write back the result to the buffer + // 6. write back the result to the buffer buffer.as_slice_mut()[byte_offset] = result_first_byte; } -/// Read 8 bits from a buffer starting at a given bit offset -#[inline] -fn get_8_bits_from_offset(buffer: &Buffer, offset_in_bits: usize) -> u8 { - let byte_offset = offset_in_bits / 8; - let bit_offset = offset_in_bits % 8; - - let first_byte = buffer.as_slice()[byte_offset] as u16; - let second_byte = if byte_offset + 1 < buffer.len() { - buffer.as_slice()[byte_offset + 1] as u16 - } else { - 0 - }; - - // Combine the two bytes into a single u16 - let combined = (second_byte << 8) | first_byte; - - // Shift right by the bit offset and mask to get the relevant 8 bits - ((combined >> bit_offset) & 0xFF) as u8 -} - +/// Read up to 8 bits from a byte slice starting at a given bit offset. +/// +/// This is similar to `get_8_bits_from_offset` but works with raw byte slices +/// and can read fewer than 8 bits. +/// +/// # Arguments +/// +/// * `slice` - The byte slice to read from +/// * `number_of_bits_to_read` - Number of bits to read (must be ≤ 8) +/// * `bit_offset` - Starting bit offset within the first byte +/// +/// # Returns +/// +/// A u8 containing the requested bits in the least significant positions #[inline] fn read_up_to_byte_from_offset( slice: &[u8], @@ -157,7 +262,7 @@ fn read_up_to_byte_from_offset( // might be one more than sizeof(u64) if the offset is in the middle of a byte let byte_len = ceil(bit_len + bit_offset, 8); // pointer to remainder bytes after all complete chunks - let base = unsafe { slice.as_ptr() }; + let base = slice.as_ptr(); let mut bits = unsafe { std::ptr::read(base) } >> bit_offset; for i in 1..byte_len { @@ -169,18 +274,29 @@ fn read_up_to_byte_from_offset( } } -/// Helper function to run the bitwise operation when we know that the left offset is byte-aligned. -/// This is the easiest case as we can do the operation directly on u64 chunks and then handle the remainder bits if any. +/// Perform bitwise binary operation on byte-aligned buffers (i.e. not offsetting into a middle of a byte). +/// +/// This is the optimized path for byte-aligned operations. It processes data in +/// u64 chunks for maximum efficiency, then handles any remainder bits. +/// +/// # Arguments +/// +/// * `left` - The left mutable buffer (must be byte-aligned) +/// * `left_offset_in_bits` - Starting bit offset in the left buffer (must be multiple of 8) +/// * `right` - The right buffer as byte slice +/// * `right_offset_in_bits` - Starting bit offset in the right buffer +/// * `len_in_bits` - Number of bits to process +/// * `op` - Binary operation to apply #[inline] -fn mutable_left_byte_aligned_bitwise_bin_op_helper( +fn mutable_buffer_byte_aligned_bitwise_bin_op_helper( left: &mut MutableBuffer, left_offset_in_bits: usize, - right: &[u8], + right: &impl BufferSupportedRhs, right_offset_in_bits: usize, len_in_bits: usize, mut op: F, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { // Must not reach here if we not byte aligned assert_eq!( @@ -189,7 +305,8 @@ fn mutable_left_byte_aligned_bitwise_bin_op_helper( "left_offset_in_bits must be byte aligned" ); - let right_chunks = BitChunks::new(right, right_offset_in_bits, len_in_bits); + // 1. Prepare the buffers + let right_chunks = BitChunks::new(right.as_slice(), right_offset_in_bits, len_in_bits); let left_buffer_mut: &mut [u8] = { assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len() * 8); @@ -212,23 +329,24 @@ fn mutable_left_byte_aligned_bitwise_bin_op_helper( // If not only remainder bytes let had_any_chunks = right_chunks_iter.len() > 0; - // Trying to read the first chunk - // we do this outside the loop so in the loop we can increment the pointer first - // and then read the value - // avoiding incrementing the pointer after the last read - if let Some(right) = right_chunks_iter.next() { - unsafe { - run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + // 2. Process complete u64 chunks + { + // Process the first chunk outside the loop to avoid incrementing + // the pointer after the last read + if let Some(right) = right_chunks_iter.next() { + unsafe { + run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + } } - } - for right in right_chunks_iter { - // Increase the pointer for the next iteration - // we are increasing the pointer before reading because we already read the first chunk above - left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; + for right in right_chunks_iter { + // Increase the pointer for the next iteration + // we are increasing the pointer before reading because we already read the first chunk above + left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; - unsafe { - run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + unsafe { + run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + } } } @@ -251,6 +369,17 @@ fn mutable_left_byte_aligned_bitwise_bin_op_helper( } } +/// Handle remainder bits (< 64 bits) for binary operations. +/// +/// This function processes the bits that don't form a complete u64 chunk, +/// ensuring that bits outside the operation range are preserved. +/// +/// # Arguments +/// +/// * `op` - Binary operation to apply +/// * `start_remainder_mut_ptr` - Pointer to the start of remainder bytes +/// * `right_remainder_bits` - Right operand bits +/// * `remainder_len` - Number of remainder bits #[inline] fn handle_mutable_buffer_remainder( op: &mut F, @@ -258,7 +387,7 @@ fn handle_mutable_buffer_remainder( right_remainder_bits: u64, remainder_len: usize, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { // Only read from mut pointer the number of remainder bits let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); @@ -270,6 +399,16 @@ fn handle_mutable_buffer_remainder( set_remainder_bits(start_remainder_mut_ptr, rem, remainder_len); } +/// Write remainder bits back to buffer while preserving bits outside the range. +/// +/// This function carefully updates only the specified bits, leaving all other +/// bits in the affected bytes unchanged. +/// +/// # Arguments +/// +/// * `start_remainder_mut_ptr` - Pointer to the start of remainder bytes +/// * `rem` - The result bits to write +/// * `remainder_len` - Number of bits to write #[inline] fn set_remainder_bits(start_remainder_mut_ptr: *mut u8, rem: u64, remainder_len: usize) { // Need to update the remainder bytes in the mutable buffer @@ -279,7 +418,7 @@ fn set_remainder_bits(start_remainder_mut_ptr: *mut u8, rem: u64, remainder_len: // to preserve the bits outside the remainder let rem = { // 1. Read the byte that we will override - let mut current = { + let current = { let last_byte_position = remainder_len / 8; let last_byte_ptr = unsafe { start_remainder_mut_ptr.add(last_byte_position) }; @@ -307,23 +446,33 @@ fn set_remainder_bits(start_remainder_mut_ptr: *mut u8, rem: u64, remainder_len: { let remainder_bytes = ceil(remainder_len, 8); - // we are counting its starting from the least significant bit, to to_le_bytes should be correct + // we are counting starting from the least significant bit, so to_le_bytes should be correct let rem = &rem.to_le_bytes()[0..remainder_bytes]; // this assumes that `[ToByteSlice]` can be copied directly // without calling `to_byte_slice` for each element, // which is correct for all ArrowNativeType implementations. - let src = rem.as_ptr() as *const u8; + let src = rem.as_ptr(); unsafe { std::ptr::copy_nonoverlapping(src, start_remainder_mut_ptr, remainder_bytes) }; } } -// Get remainder bits from a pointer and length in bits +/// Read remainder bits from a pointer. +/// +/// Reads the specified number of bits from memory and returns them as a u64. +/// +/// # Arguments +/// +/// * `remainder_ptr` - Pointer to the start of the bits +/// * `remainder_len` - Number of bits to read (must be < 64) +/// +/// # Returns +/// +/// A u64 containing the bits in the least significant positions #[inline] fn get_remainder_bits(remainder_ptr: *const u8, remainder_len: usize) -> u64 { let bit_len = remainder_len; // number of bytes to read - // might be one more than sizeof(u64) if the offset is in the middle of a byte let byte_len = ceil(bit_len, 8); // pointer to remainder bytes after all complete chunks let base = remainder_ptr; @@ -337,14 +486,28 @@ fn get_remainder_bits(remainder_ptr: *const u8, remainder_len: usize) -> u64 { bits & ((1 << bit_len) - 1) } -// TODO - find a better name +/// Apply a binary operation to a u64 in memory. +/// +/// Reads a u64 from the pointer, applies the operation with the right operand, +/// and writes the result back. Handles endianness correctly for bit-packed buffers. +/// +/// # Safety +/// +/// The pointer must be valid for reads and writes of u64 values. +/// Unaligned access is handled correctly via `read_unaligned` and `write_unaligned`. +/// +/// # Arguments +/// +/// * `op` - Binary operation to apply +/// * `left_buffer_mut_ptr` - Pointer to the left operand u64 +/// * `right` - Right operand value #[inline] unsafe fn run_op_on_mutable_pointer_and_single_value( op: &mut F, left_buffer_mut_ptr: *mut u64, right: u64, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { // 1. Read the current value from the mutable buffer // @@ -356,12 +519,20 @@ unsafe fn run_op_on_mutable_pointer_and_single_value( let combined = op(current, right); // 3. Write the new value back to the mutable buffer - // TODO - should write with to_le()? because we already read as to_le() unsafe { std::ptr::write_unaligned(left_buffer_mut_ptr, combined) }; } -/// Helper function to run the bitwise operation when we know that the left offset is byte-aligned. -/// This is the easiest case as we can do the operation directly on u64 chunks and then handle the remainder bits if any. +/// Perform bitwise unary operation on byte-aligned buffer. +/// +/// This is the optimized path for byte-aligned unary operations. It processes data in +/// u64 chunks for maximum efficiency, then handles any remainder bits. +/// +/// # Arguments +/// +/// * `left` - The mutable buffer (must be byte-aligned) +/// * `left_offset_in_bits` - Starting bit offset (must be multiple of 8) +/// * `len_in_bits` - Number of bits to process +/// * `op` - Unary operation to apply (e.g., `|a| !a`) #[inline] fn mutable_byte_aligned_bitwise_unary_op_helper( left: &mut MutableBuffer, @@ -369,7 +540,7 @@ fn mutable_byte_aligned_bitwise_unary_op_helper( len_in_bits: usize, mut op: F, ) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { // Must not reach here if we not byte aligned assert_eq!( @@ -396,10 +567,7 @@ fn mutable_byte_aligned_bitwise_unary_op_helper( // If not only remainder bytes let had_any_chunks = number_of_u64_chunks > 0; - // Trying to read the first chunk - // we do this outside the loop so in the loop we can increment the pointer first - // and then read the value - // avoiding incrementing the pointer after the last read + // Process the first chunk outside the loop if had_any_chunks { unsafe { run_op_on_mutable_pointer(&mut op, left_buffer_mut_u64_ptr); @@ -408,7 +576,6 @@ fn mutable_byte_aligned_bitwise_unary_op_helper( for _ in 1..number_of_u64_chunks { // Increase the pointer for the next iteration - // we are increasing the pointer before reading because we already read the first chunk above left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; unsafe { @@ -419,7 +586,7 @@ fn mutable_byte_aligned_bitwise_unary_op_helper( // Handle remainder bits if any if remainder_len > 0 { { - // If we had any chunks we only advance the pointer at the start. + // If we had any chunks we only advance the pointer at the start, // so we need to advance it again if we have a remainder let advance_pointer_count = if had_any_chunks { 1 } else { 0 }; left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(advance_pointer_count) } @@ -430,33 +597,53 @@ fn mutable_byte_aligned_bitwise_unary_op_helper( } } -// TODO - find a better name +/// Apply a unary operation to a u64 in memory. +/// +/// Reads a u64 from the pointer, applies the operation, and writes the result back. +/// +/// # Safety +/// +/// The pointer must be valid for reads and writes of u64 values. +/// +/// # Arguments +/// +/// * `op` - Unary operation to apply +/// * `buffer_mut_ptr` - Pointer to the u64 #[inline] -unsafe fn run_op_on_mutable_pointer(op: &mut F, left_buffer_mut_ptr: *mut u64) +unsafe fn run_op_on_mutable_pointer(op: &mut F, buffer_mut_ptr: *mut u64) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { // 1. Read the current value from the mutable buffer // // bit-packed buffers are stored starting with the least-significant byte first // so when reading as u64 on a big-endian machine, the bytes need to be swapped - let current = unsafe { std::ptr::read_unaligned(left_buffer_mut_ptr).to_le() }; + let current = unsafe { std::ptr::read_unaligned(buffer_mut_ptr).to_le() }; // 2. Get the new value by applying the operation let result = op(current); // 3. Write the new value back to the mutable buffer - // TODO - should write with to_le()? because we already read as to_le() - unsafe { std::ptr::write_unaligned(left_buffer_mut_ptr, result) }; + unsafe { std::ptr::write_unaligned(buffer_mut_ptr, result) }; } +/// Handle remainder bits (< 64 bits) for unary operations. +/// +/// This function processes the bits that don't form a complete u64 chunk, +/// ensuring that bits outside the operation range are preserved. +/// +/// # Arguments +/// +/// * `op` - Unary operation to apply +/// * `start_remainder_mut_ptr` - Pointer to the start of remainder bytes +/// * `remainder_len` - Number of remainder bits #[inline] fn handle_mutable_buffer_remainder_unary( op: &mut F, start_remainder_mut_ptr: *mut u8, remainder_len: usize, ) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { // Only read from mut pointer the number of remainder bits let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); @@ -468,168 +655,201 @@ fn handle_mutable_buffer_remainder_unary( set_remainder_bits(start_remainder_mut_ptr, rem, remainder_len); } -/// Apply a bitwise operation `op` to the passed [`MutableBuffer`] and update it -/// The input is treated as a bitmap, meaning that offset and length are specified in number of bits. -pub(crate) fn mutable_bitwise_unary_op_helper( - buffer: &mut MutableBuffer, +/// Apply a bitwise operation to a mutable buffer and update it in-place. +/// +/// This is the main entry point for unary operations. It handles both byte-aligned +/// and non-byte-aligned cases. +/// +/// The input is treated as a bitmap, meaning that offset and length are specified +/// in number of bits. +/// +/// # Arguments +/// +/// * `buffer` - The mutable buffer to modify +/// * `offset_in_bits` - Starting bit offset +/// * `len_in_bits` - Number of bits to process +/// * `op` - Unary operation to apply (e.g., `|a| !a`) +#[allow( + private_bounds, + reason = "MutableOpsBufferSupportedLhs exposes the inner internals which is the implementor choice and we dont want to leak internals" +)] +pub fn mutable_bitwise_unary_op_helper( + buffer: &mut impl MutableOpsBufferSupportedLhs, offset_in_bits: usize, len_in_bits: usize, mut op: F, ) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { if len_in_bits == 0 { return; } + let mutable_buffer = buffer.inner_mutable_buffer(); + + let mutable_buffer_len = mutable_buffer.len(); + let mutable_buffer_cap = mutable_buffer.capacity(); + // offset inside a byte let left_bit_offset = offset_in_bits % 8; let is_mutable_buffer_byte_aligned = left_bit_offset == 0; if is_mutable_buffer_byte_aligned { - mutable_byte_aligned_bitwise_unary_op_helper(buffer, offset_in_bits, len_in_bits, op); + mutable_byte_aligned_bitwise_unary_op_helper(mutable_buffer, offset_in_bits, len_in_bits, op); } else { // If we are not byte aligned we will read the first few bits let bits_to_next_byte = 8 - left_bit_offset; - align_to_byte(&mut op, buffer, offset_in_bits); + align_to_byte(&mut op, mutable_buffer, offset_in_bits); let offset_in_bits = offset_in_bits + bits_to_next_byte; let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); if len_in_bits == 0 { + // Making sure that our guarantee that the length and capacity of the mutable buffer + // will not change is upheld + assert_eq!(mutable_buffer.len(), mutable_buffer_len, "The length of the mutable buffer must not change"); + assert_eq!(mutable_buffer.capacity(), mutable_buffer_cap, "The capacity of the mutable buffer must not change"); + return; } // We are now byte aligned - mutable_byte_aligned_bitwise_unary_op_helper(buffer, offset_in_bits, len_in_bits, op); + mutable_byte_aligned_bitwise_unary_op_helper(mutable_buffer, offset_in_bits, len_in_bits, op); } -} -/// Apply a bitwise and to two inputs and return the result as a Buffer. -/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub(crate) fn left_mutable_buffer_bin_and( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &Buffer, - right_offset_in_bits: usize, - len_in_bits: usize, -) { - left_mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right.as_slice(), - right_offset_in_bits, - len_in_bits, - |a, b| a & b, - ) + // Making sure that our guarantee that the length and capacity of the mutable buffer + // will not change is upheld + assert_eq!(mutable_buffer.len(), mutable_buffer_len, "The length of the mutable buffer must not change"); + assert_eq!(mutable_buffer.capacity(), mutable_buffer_cap, "The capacity of the mutable buffer must not change"); } -/// Apply a bitwise and to two inputs and return the result as a Buffer. -/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub(crate) fn both_mutable_buffer_bin_and( - left: &mut MutableBuffer, +/// Apply a bitwise AND operation to two buffers. +/// +/// The left buffer (mutable) is modified in-place to contain the result. +/// The inputs are treated as bitmaps, meaning that offsets and length are +/// specified in number of bits. +/// +/// # Arguments +/// +/// * `left` - The left mutable buffer (will be modified) +/// * `left_offset_in_bits` - Starting bit offset in the left buffer +/// * `right` - The right buffer +/// * `right_offset_in_bits` - Starting bit offset in the right buffer +/// * `len_in_bits` - Number of bits to process +#[allow( + private_bounds, + reason = "MutableOpsBufferSupportedLhs and BufferSupportedRhs exposes the inner internals which is the implementor choice and we dont want to leak internals" +)] +pub fn mutable_buffer_bin_and( + left: &mut impl MutableOpsBufferSupportedLhs, left_offset_in_bits: usize, - right: &MutableBuffer, + right: &impl BufferSupportedRhs, right_offset_in_bits: usize, len_in_bits: usize, ) { - left_mutable_bitwise_bin_op_helper( + mutable_bitwise_bin_op_helper( left, left_offset_in_bits, - right.as_slice(), + right, right_offset_in_bits, len_in_bits, |a, b| a & b, ) } -/// Apply a bitwise or to two inputs and return the result as a Buffer. -/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub(crate) fn left_mutable_buffer_bin_or( - left: &mut MutableBuffer, +/// Apply a bitwise OR operation to two buffers. +/// +/// The left buffer (mutable) is modified in-place to contain the result. +/// The inputs are treated as bitmaps, meaning that offsets and length are +/// specified in number of bits. +/// +/// # Arguments +/// +/// * `left` - The left mutable buffer (will be modified) +/// * `left_offset_in_bits` - Starting bit offset in the left buffer +/// * `right` - The right buffer +/// * `right_offset_in_bits` - Starting bit offset in the right buffer +/// * `len_in_bits` - Number of bits to process +#[allow( + private_bounds, + reason = "MutableOpsBufferSupportedLhs and BufferSupportedRhs exposes the inner internals which is the implementor choice and we dont want to leak internals" +)] +pub fn mutable_buffer_bin_or( + left: &mut impl MutableOpsBufferSupportedLhs, left_offset_in_bits: usize, - right: &Buffer, + right: &impl BufferSupportedRhs, right_offset_in_bits: usize, len_in_bits: usize, ) { - left_mutable_bitwise_bin_op_helper( + mutable_bitwise_bin_op_helper( left, left_offset_in_bits, - right.as_slice(), + right, right_offset_in_bits, len_in_bits, |a, b| a | b, ) } -/// Apply a bitwise or to two inputs and return the result as a Buffer. -/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub(crate) fn both_mutable_buffer_bin_or( - left: &mut MutableBuffer, +/// Apply a bitwise XOR operation to two buffers. +/// +/// The left buffer (mutable) is modified in-place to contain the result. +/// The inputs are treated as bitmaps, meaning that offsets and length are +/// specified in number of bits. +/// +/// # Arguments +/// +/// * `left` - The left mutable buffer (will be modified) +/// * `left_offset_in_bits` - Starting bit offset in the left buffer +/// * `right` - The right buffer +/// * `right_offset_in_bits` - Starting bit offset in the right buffer +/// * `len_in_bits` - Number of bits to process +#[allow( + private_bounds, + reason = "MutableOpsBufferSupportedLhs and BufferSupportedRhs exposes the inner internals which is the implementor choice and we dont want to leak internals" +)] +pub fn mutable_buffer_bin_xor( + left: &mut impl MutableOpsBufferSupportedLhs, left_offset_in_bits: usize, - right: &MutableBuffer, + right: &impl BufferSupportedRhs, right_offset_in_bits: usize, len_in_bits: usize, ) { - left_mutable_bitwise_bin_op_helper( + mutable_bitwise_bin_op_helper( left, left_offset_in_bits, - right.as_slice(), - right_offset_in_bits, - len_in_bits, - |a, b| a | b, - ) -} - -/// Apply a bitwise xor to two inputs and return the result as a Buffer. -/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub(crate) fn left_mutable_buffer_bin_xor( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &Buffer, - right_offset_in_bits: usize, - len_in_bits: usize, -) { - left_mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right.as_slice(), + right, right_offset_in_bits, len_in_bits, |a, b| a ^ b, ) } -/// Apply a bitwise xor to two inputs and return the result as a Buffer. -/// The inputs are treated as bitmaps, meaning that offsets and length are specified in number of bits. -pub(crate) fn both_mutable_buffer_bin_xor( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - right: &MutableBuffer, - right_offset_in_bits: usize, - len_in_bits: usize, -) { - left_mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right.as_slice(), - right_offset_in_bits, - len_in_bits, - |a, b| a ^ b, - ) -} - -/// Apply a bitwise not to one input and return the result as a Buffer. -/// The input is treated as a bitmap, meaning that offset and length are specified in number of bits. -pub(crate) fn mutable_buffer_unary_not( - left: &mut MutableBuffer, +/// Apply a bitwise NOT operation to the passed buffer. +/// +/// The buffer (mutable) is modified in-place to contain the result. +/// The input is treated as bitmap, meaning that offsets and length are +/// specified in number of bits. +/// +/// # Arguments +/// +/// * `buffer` - The mutable buffer (will be modified) +/// * `offset_in_bits` - Starting bit offset in the buffer +/// * `len_in_bits` - Number of bits to process +#[allow( + private_bounds, + reason = "MutableOpsBufferSupportedLhs exposes the inner internals which is the implementor choice and we dont want to leak internals" +)] +pub fn mutable_buffer_unary_not( + buffer: &mut impl MutableOpsBufferSupportedLhs, offset_in_bits: usize, len_in_bits: usize, ) { - mutable_bitwise_unary_op_helper(left, offset_in_bits, len_in_bits, |a| !a) + mutable_bitwise_unary_op_helper(buffer, offset_in_bits, len_in_bits, |a| !a) } + #[cfg(test)] mod tests { use crate::bit_iterator::BitIterator; @@ -647,7 +867,7 @@ mod tests { F: FnMut(u64, u64) -> u64, G: FnMut(bool, bool) -> bool, { - let mut left_buffer = BooleanBufferBuilder::from(left_data).into_inner(); + let mut left_buffer = BooleanBufferBuilder::from(left_data); let right_buffer = BooleanBuffer::from(right_data); let expected: Vec = left_data @@ -658,10 +878,10 @@ mod tests { .map(|(l, r)| expected_op(*l, *r)) .collect(); - super::left_mutable_bitwise_bin_op_helper( + super::mutable_bitwise_bin_op_helper( &mut left_buffer, left_offset_in_bits, - right_buffer.values(), + right_buffer.inner(), right_offset_in_bits, len_in_bits, op, @@ -687,7 +907,7 @@ mod tests { F: FnMut(u64) -> u64, G: FnMut(bool) -> bool, { - let mut buffer = BooleanBufferBuilder::from(data).into_inner(); + let mut buffer = BooleanBufferBuilder::from(data); let expected: Vec = data .iter() diff --git a/arrow-buffer/src/builder/boolean.rs b/arrow-buffer/src/builder/boolean.rs index a1318b46cad8..3600fec9adf9 100644 --- a/arrow-buffer/src/builder/boolean.rs +++ b/arrow-buffer/src/builder/boolean.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::{BooleanBuffer, Buffer, MutableBuffer, bit_mask, bit_util, left_mutable_buffer_bin_and, both_mutable_buffer_bin_and, left_mutable_buffer_bin_or, both_mutable_buffer_bin_or, left_mutable_buffer_bin_xor, both_mutable_buffer_bin_xor, mutable_buffer_unary_not}; +use crate::{BooleanBuffer, Buffer, MutableBuffer, bit_mask, bit_util, mutable_buffer_bin_and, mutable_buffer_bin_or, mutable_buffer_bin_xor, mutable_buffer_unary_not, MutableOpsBufferSupportedLhs}; use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not, Range}; /// Builder for [`BooleanBuffer`] @@ -256,33 +256,13 @@ impl BooleanBufferBuilder { pub fn finish_cloned(&self) -> BooleanBuffer { BooleanBuffer::new(Buffer::from_slice_ref(self.as_slice()), 0, self.len) } +} - /// Returns a reference to the inner [`MutableBuffer`] - /// - /// only in tests and not public API to avoid misuse as the length of the buffer - /// is not updated when modifying the inner buffer directly - #[cfg(test)] - pub(crate) fn inner_mut(&mut self) -> &mut MutableBuffer { +/// This trait is not public API so it does not leak the inner mutable buffer +impl MutableOpsBufferSupportedLhs for BooleanBufferBuilder { + fn inner_mutable_buffer(&mut self) -> &mut MutableBuffer { &mut self.buffer } - - /// Get the inner [`MutableBuffer`] - /// - /// Note: it might be larger than the actual length of initialized bits - /// as the bits are packed. - pub fn into_inner(self) -> MutableBuffer { - self.buffer - } - - - pub fn fixed_slice(&mut self, range: Range) -> BooleanBuffer { - assert!(range.end <= self.len); - BooleanBuffer::new( - Buffer::from_slice_ref(self.as_slice()), - range.start, - range.end - range.start, - ) - } } impl Not for BooleanBufferBuilder { @@ -321,7 +301,7 @@ impl BitAndAssign<&BooleanBuffer> for BooleanBufferBuilder { fn bitand_assign(&mut self, rhs: &BooleanBuffer) { assert_eq!(self.len, rhs.len()); - left_mutable_buffer_bin_and(&mut self.buffer, 0, &rhs.inner(), rhs.offset(), self.len); + mutable_buffer_bin_and(&mut self.buffer, 0, rhs.inner(), rhs.offset(), self.len); } } @@ -329,7 +309,7 @@ impl BitAndAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { fn bitand_assign(&mut self, rhs: &BooleanBufferBuilder) { assert_eq!(self.len, rhs.len()); - both_mutable_buffer_bin_and(&mut self.buffer, 0, &rhs.buffer, 0, self.len); + mutable_buffer_bin_and(&mut self.buffer, 0, &rhs.buffer, 0, self.len); } } @@ -357,7 +337,7 @@ impl BitOrAssign<&BooleanBuffer> for BooleanBufferBuilder { fn bitor_assign(&mut self, rhs: &BooleanBuffer) { assert_eq!(self.len, rhs.len()); - left_mutable_buffer_bin_or(&mut self.buffer, 0, &rhs.inner(), rhs.offset(), self.len); + mutable_buffer_bin_or(&mut self.buffer, 0, rhs.inner(), rhs.offset(), self.len); } } @@ -365,7 +345,7 @@ impl BitOrAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { fn bitor_assign(&mut self, rhs: &BooleanBufferBuilder) { assert_eq!(self.len, rhs.len()); - both_mutable_buffer_bin_or(&mut self.buffer, 0, &rhs.buffer, 0, self.len); + mutable_buffer_bin_or(&mut self.buffer, 0, &rhs.buffer, 0, self.len); } } @@ -393,7 +373,7 @@ impl BitXorAssign<&BooleanBuffer> for BooleanBufferBuilder { fn bitxor_assign(&mut self, rhs: &BooleanBuffer) { assert_eq!(self.len, rhs.len()); - left_mutable_buffer_bin_xor(&mut self.buffer, 0, &rhs.inner(), rhs.offset(), self.len); + mutable_buffer_bin_xor(&mut self.buffer, 0, rhs.inner(), rhs.offset(), self.len); } } @@ -401,7 +381,7 @@ impl BitXorAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { fn bitxor_assign(&mut self, rhs: &BooleanBufferBuilder) { assert_eq!(self.len, rhs.len()); - both_mutable_buffer_bin_xor(&mut self.buffer, 0, &rhs.buffer, 0, self.len); + mutable_buffer_bin_xor(&mut self.buffer, 0, &rhs.buffer, 0, self.len); } } From da03628b8e060e0e0c1c8eb2325686d4f1fc9fb9 Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Wed, 15 Oct 2025 10:30:19 +0300 Subject: [PATCH 08/15] format --- arrow-buffer/src/buffer/mutable.rs | 10 ++- arrow-buffer/src/buffer/mutable_ops.rs | 95 +++++++++++++++------ arrow-buffer/src/builder/boolean.rs | 10 ++- arrow-buffer/src/util/bit_chunk_iterator.rs | 6 +- 4 files changed, 85 insertions(+), 36 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable.rs b/arrow-buffer/src/buffer/mutable.rs index bdfd790deec0..6f7a90d573ef 100644 --- a/arrow-buffer/src/buffer/mutable.rs +++ b/arrow-buffer/src/buffer/mutable.rs @@ -20,14 +20,18 @@ use std::mem; use std::ptr::NonNull; use crate::alloc::{ALIGNMENT, Deallocation}; -use crate::{bytes::Bytes, native::{ArrowNativeType, ToByteSlice}, util::bit_util}; +use crate::{ + bytes::Bytes, + native::{ArrowNativeType, ToByteSlice}, + util::bit_util, +}; +use super::Buffer; +use crate::bit_chunk_iterator::{BitChunks, UnalignedBitChunk}; #[cfg(feature = "pool")] use crate::pool::{MemoryPool, MemoryReservation}; #[cfg(feature = "pool")] use std::sync::Mutex; -use crate::bit_chunk_iterator::{BitChunks, UnalignedBitChunk}; -use super::Buffer; /// A [`MutableBuffer`] is Arrow's interface to build a [`Buffer`] out of items or slices of items. /// diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index 8887272b3079..4fb430d93463 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -16,21 +16,20 @@ // under the License. use super::{Buffer, MutableBuffer}; -use crate::bit_chunk_iterator::BitChunks; use crate::BooleanBufferBuilder; +use crate::bit_chunk_iterator::BitChunks; use crate::util::bit_util::ceil; - /// What can be used as the right-hand side (RHS) buffer in mutable operations. -/// +/// /// this is not mutated. -/// +/// /// # Implementation notes -/// +/// /// ## Why `pub(crate)`? /// This is because we don't want this trait to expose the inner buffer to the public. /// this is the trait implementor choice. -/// +/// pub(crate) trait BufferSupportedRhs { fn as_slice(&self) -> &[u8]; } @@ -59,7 +58,7 @@ impl BufferSupportedRhs for BooleanBufferBuilder { /// 1. It will not change the length of the buffer. /// /// # Implementation notes -/// +/// /// ## Why is this trait `pub(crate)`? /// Because we don't wanna expose the inner mutable buffer to the public. /// as this is the choice of the implementor of the trait and sometimes it is not desirable @@ -114,7 +113,7 @@ pub fn mutable_bitwise_bin_op_helper( len_in_bits: usize, mut op: F, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { if len_in_bits == 0 { return; @@ -169,8 +168,16 @@ pub fn mutable_bitwise_bin_op_helper( if len_in_bits == 0 { // Making sure that our guarantee that the length and capacity of the mutable buffer // will not change is upheld - assert_eq!(mutable_buffer.len(), mutable_buffer_len, "The length of the mutable buffer must not change"); - assert_eq!(mutable_buffer.capacity(), mutable_buffer_cap, "The capacity of the mutable buffer must not change"); + assert_eq!( + mutable_buffer.len(), + mutable_buffer_len, + "The length of the mutable buffer must not change" + ); + assert_eq!( + mutable_buffer.capacity(), + mutable_buffer_cap, + "The capacity of the mutable buffer must not change" + ); return; } @@ -188,8 +195,16 @@ pub fn mutable_bitwise_bin_op_helper( // Making sure that our guarantee that the length and capacity of the mutable buffer // will not change is upheld - assert_eq!(mutable_buffer.len(), mutable_buffer_len, "The length of the mutable buffer must not change"); - assert_eq!(mutable_buffer.capacity(), mutable_buffer_cap, "The capacity of the mutable buffer must not change"); + assert_eq!( + mutable_buffer.len(), + mutable_buffer_len, + "The length of the mutable buffer must not change" + ); + assert_eq!( + mutable_buffer.capacity(), + mutable_buffer_cap, + "The capacity of the mutable buffer must not change" + ); } /// Align to byte boundary by applying operation to bits before the next byte boundary. @@ -204,7 +219,7 @@ pub fn mutable_bitwise_bin_op_helper( /// * `offset_in_bits` - Starting bit offset (not byte-aligned) fn align_to_byte(op: &mut F, buffer: &mut MutableBuffer, offset_in_bits: usize) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { let byte_offset = offset_in_bits / 8; let bit_offset = offset_in_bits % 8; @@ -226,7 +241,7 @@ where let mask_for_first_bit_offset = (1 << bit_offset) - 1; let result_first_byte = - (first_byte & mask_for_first_bit_offset) | (result_first_byte & !mask_for_first_bit_offset); + (first_byte & mask_for_first_bit_offset) | (result_first_byte & !mask_for_first_bit_offset); // 6. write back the result to the buffer buffer.as_slice_mut()[byte_offset] = result_first_byte; @@ -296,7 +311,7 @@ fn mutable_buffer_byte_aligned_bitwise_bin_op_helper( len_in_bits: usize, mut op: F, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { // Must not reach here if we not byte aligned assert_eq!( @@ -387,7 +402,7 @@ fn handle_mutable_buffer_remainder( right_remainder_bits: u64, remainder_len: usize, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { // Only read from mut pointer the number of remainder bits let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); @@ -507,7 +522,7 @@ unsafe fn run_op_on_mutable_pointer_and_single_value( left_buffer_mut_ptr: *mut u64, right: u64, ) where - F: FnMut(u64, u64) -> u64, + F: FnMut(u64, u64) -> u64, { // 1. Read the current value from the mutable buffer // @@ -540,7 +555,7 @@ fn mutable_byte_aligned_bitwise_unary_op_helper( len_in_bits: usize, mut op: F, ) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { // Must not reach here if we not byte aligned assert_eq!( @@ -612,7 +627,7 @@ fn mutable_byte_aligned_bitwise_unary_op_helper( #[inline] unsafe fn run_op_on_mutable_pointer(op: &mut F, buffer_mut_ptr: *mut u64) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { // 1. Read the current value from the mutable buffer // @@ -643,7 +658,7 @@ fn handle_mutable_buffer_remainder_unary( start_remainder_mut_ptr: *mut u8, remainder_len: usize, ) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { // Only read from mut pointer the number of remainder bits let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); @@ -679,7 +694,7 @@ pub fn mutable_bitwise_unary_op_helper( len_in_bits: usize, mut op: F, ) where - F: FnMut(u64) -> u64, + F: FnMut(u64) -> u64, { if len_in_bits == 0 { return; @@ -696,7 +711,12 @@ pub fn mutable_bitwise_unary_op_helper( let is_mutable_buffer_byte_aligned = left_bit_offset == 0; if is_mutable_buffer_byte_aligned { - mutable_byte_aligned_bitwise_unary_op_helper(mutable_buffer, offset_in_bits, len_in_bits, op); + mutable_byte_aligned_bitwise_unary_op_helper( + mutable_buffer, + offset_in_bits, + len_in_bits, + op, + ); } else { // If we are not byte aligned we will read the first few bits let bits_to_next_byte = 8 - left_bit_offset; @@ -709,20 +729,41 @@ pub fn mutable_bitwise_unary_op_helper( if len_in_bits == 0 { // Making sure that our guarantee that the length and capacity of the mutable buffer // will not change is upheld - assert_eq!(mutable_buffer.len(), mutable_buffer_len, "The length of the mutable buffer must not change"); - assert_eq!(mutable_buffer.capacity(), mutable_buffer_cap, "The capacity of the mutable buffer must not change"); + assert_eq!( + mutable_buffer.len(), + mutable_buffer_len, + "The length of the mutable buffer must not change" + ); + assert_eq!( + mutable_buffer.capacity(), + mutable_buffer_cap, + "The capacity of the mutable buffer must not change" + ); return; } // We are now byte aligned - mutable_byte_aligned_bitwise_unary_op_helper(mutable_buffer, offset_in_bits, len_in_bits, op); + mutable_byte_aligned_bitwise_unary_op_helper( + mutable_buffer, + offset_in_bits, + len_in_bits, + op, + ); } // Making sure that our guarantee that the length and capacity of the mutable buffer // will not change is upheld - assert_eq!(mutable_buffer.len(), mutable_buffer_len, "The length of the mutable buffer must not change"); - assert_eq!(mutable_buffer.capacity(), mutable_buffer_cap, "The capacity of the mutable buffer must not change"); + assert_eq!( + mutable_buffer.len(), + mutable_buffer_len, + "The length of the mutable buffer must not change" + ); + assert_eq!( + mutable_buffer.capacity(), + mutable_buffer_cap, + "The capacity of the mutable buffer must not change" + ); } /// Apply a bitwise AND operation to two buffers. diff --git a/arrow-buffer/src/builder/boolean.rs b/arrow-buffer/src/builder/boolean.rs index 3600fec9adf9..ff8a1f66a303 100644 --- a/arrow-buffer/src/builder/boolean.rs +++ b/arrow-buffer/src/builder/boolean.rs @@ -15,7 +15,11 @@ // specific language governing permissions and limitations // under the License. -use crate::{BooleanBuffer, Buffer, MutableBuffer, bit_mask, bit_util, mutable_buffer_bin_and, mutable_buffer_bin_or, mutable_buffer_bin_xor, mutable_buffer_unary_not, MutableOpsBufferSupportedLhs}; +use crate::{ + BooleanBuffer, Buffer, MutableBuffer, MutableOpsBufferSupportedLhs, bit_mask, bit_util, + mutable_buffer_bin_and, mutable_buffer_bin_or, mutable_buffer_bin_xor, + mutable_buffer_unary_not, +}; use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not, Range}; /// Builder for [`BooleanBuffer`] @@ -282,7 +286,7 @@ impl BitAnd<&BooleanBuffer> for BooleanBufferBuilder { fn bitand(mut self, rhs: &BooleanBuffer) -> Self::Output { self &= rhs; - + self } } @@ -404,7 +408,7 @@ impl From<&[bool]> for BooleanBufferBuilder { fn from(source: &[bool]) -> Self { let mut builder = BooleanBufferBuilder::new(source.len()); builder.append_slice(source); - + builder } } diff --git a/arrow-buffer/src/util/bit_chunk_iterator.rs b/arrow-buffer/src/util/bit_chunk_iterator.rs index a4a0ede80454..ea8e8f472ace 100644 --- a/arrow-buffer/src/util/bit_chunk_iterator.rs +++ b/arrow-buffer/src/util/bit_chunk_iterator.rs @@ -276,8 +276,8 @@ impl<'a> BitChunks<'a> { // pointer to remainder bytes after all complete chunks let base = unsafe { self.buffer - .as_ptr() - .add(self.chunk_len * std::mem::size_of::()) + .as_ptr() + .add(self.chunk_len * std::mem::size_of::()) }; let mut bits = unsafe { std::ptr::read(base) } as u64 >> bit_offset; @@ -343,7 +343,7 @@ impl Iterator for BitChunkIterator<'_> { // the constructor ensures that bit_offset is in 0..8 // that means we need to read at most one additional byte to fill in the high bits let next = - unsafe { std::ptr::read_unaligned(raw_data.add(index + 1) as *const u8) as u64 }; + unsafe { std::ptr::read_unaligned(raw_data.add(index + 1) as *const u8) as u64 }; (current >> bit_offset) | (next << (64 - bit_offset)) }; From 652a25698d88294fddf45a361f6a6c1cd962bc11 Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Wed, 15 Oct 2025 10:32:19 +0300 Subject: [PATCH 09/15] revert changes --- arrow-buffer/src/buffer/mutable.rs | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable.rs b/arrow-buffer/src/buffer/mutable.rs index 6f7a90d573ef..93d9d6b9ad84 100644 --- a/arrow-buffer/src/buffer/mutable.rs +++ b/arrow-buffer/src/buffer/mutable.rs @@ -26,13 +26,13 @@ use crate::{ util::bit_util, }; -use super::Buffer; -use crate::bit_chunk_iterator::{BitChunks, UnalignedBitChunk}; #[cfg(feature = "pool")] use crate::pool::{MemoryPool, MemoryReservation}; #[cfg(feature = "pool")] use std::sync::Mutex; +use super::Buffer; + /// A [`MutableBuffer`] is Arrow's interface to build a [`Buffer`] out of items or slices of items. /// /// [`Buffer`]s created from [`MutableBuffer`] (via `into`) are guaranteed to have its pointer aligned @@ -515,19 +515,6 @@ impl MutableBuffer { buffer } - /// Returns a `BitChunks` instance which can be used to iterate over this buffers bits - /// in larger chunks and starting at arbitrary bit offsets. - /// Note that both `offset` and `length` are measured in bits. - pub fn bit_chunks(&self, offset: usize, len: usize) -> BitChunks<'_> { - BitChunks::new(self.as_slice(), offset, len) - } - - /// Returns the number of 1-bits in this buffer, starting from `offset` with `length` bits - /// inspected. Note that both `offset` and `length` are measured in bits. - pub fn count_set_bits_offset(&self, offset: usize, len: usize) -> usize { - UnalignedBitChunk::new(self.as_slice(), offset, len).count_ones() - } - /// Register this [`MutableBuffer`] with the provided [`MemoryPool`] /// /// This claims the memory used by this buffer in the pool, allowing for From 0c29f0ed8d53a967f77890914547768cca4991a9 Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Wed, 15 Oct 2025 12:14:15 +0300 Subject: [PATCH 10/15] fix validation --- arrow-buffer/src/buffer/mutable_ops.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index 4fb430d93463..c5595dd29949 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -323,7 +323,7 @@ fn mutable_buffer_byte_aligned_bitwise_bin_op_helper( // 1. Prepare the buffers let right_chunks = BitChunks::new(right.as_slice(), right_offset_in_bits, len_in_bits); let left_buffer_mut: &mut [u8] = { - assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len() * 8); + assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len()); let byte_offset = left_offset_in_bits / 8; @@ -568,7 +568,7 @@ fn mutable_byte_aligned_bitwise_unary_op_helper( let remainder_len = len_in_bits % 64; let left_buffer_mut: &mut [u8] = { - assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len() * 8); + assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len()); let byte_offset = left_offset_in_bits / 8; From bcd4863769c42cbe4bd10ad696729fe4cf23cfbd Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Wed, 15 Oct 2025 16:04:33 +0300 Subject: [PATCH 11/15] remove many unsafe and cleanup --- arrow-buffer/src/buffer/mutable_ops.rs | 401 +++++++++++++------------ 1 file changed, 214 insertions(+), 187 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index c5595dd29949..89b22579daf7 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -18,7 +18,7 @@ use super::{Buffer, MutableBuffer}; use crate::BooleanBufferBuilder; use crate::bit_chunk_iterator::BitChunks; -use crate::util::bit_util::ceil; +use crate::util::bit_util; /// What can be used as the right-hand side (RHS) buffer in mutable operations. /// @@ -268,25 +268,21 @@ fn read_up_to_byte_from_offset( bit_offset: usize, ) -> u8 { assert!(number_of_bits_to_read <= 8); + assert_ne!(number_of_bits_to_read, 0); + assert_ne!(slice.len(), 0); + + let number_of_bytes_to_read = bit_util::ceil(number_of_bits_to_read + bit_offset, 8); - let bit_len = number_of_bits_to_read; - if bit_len == 0 { - 0 - } else { // number of bytes to read // might be one more than sizeof(u64) if the offset is in the middle of a byte - let byte_len = ceil(bit_len + bit_offset, 8); - // pointer to remainder bytes after all complete chunks - let base = slice.as_ptr(); - - let mut bits = unsafe { std::ptr::read(base) } >> bit_offset; - for i in 1..byte_len { - let byte = unsafe { std::ptr::read(base.add(i)) }; - bits |= (byte) << (i * 8 - bit_offset); + assert!(slice.len() >= number_of_bytes_to_read); + + let mut bits = slice[0] >> bit_offset; + for (i, &byte) in slice.iter().take(number_of_bytes_to_read).enumerate().skip(1) { + bits |= byte << (i * 8 - bit_offset); } - bits & ((1 << bit_len) - 1) - } + bits & ((1 << number_of_bits_to_read) - 1) } /// Perform bitwise binary operation on byte-aligned buffers (i.e. not offsetting into a middle of a byte). @@ -321,66 +317,199 @@ fn mutable_buffer_byte_aligned_bitwise_bin_op_helper( ); // 1. Prepare the buffers + let (complete_u64_chunks, remainder_bytes) = + U64UnalignedSlice::split(left, left_offset_in_bits, len_in_bits); + let right_chunks = BitChunks::new(right.as_slice(), right_offset_in_bits, len_in_bits); - let left_buffer_mut: &mut [u8] = { - assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len()); + assert_eq!( + bit_util::ceil(right_chunks.remainder_len(), 8), + remainder_bytes.len() + ); - let byte_offset = left_offset_in_bits / 8; + let right_chunks_iter = right_chunks.iter(); + assert_eq!(right_chunks_iter.len(), complete_u64_chunks.len()); - // number of complete u64 chunks - let chunk_len = len_in_bits / 64; + // 2. Process complete u64 chunks + complete_u64_chunks.zip_modify(right_chunks_iter, &mut op); - assert_eq!(right_chunks.chunk_len(), chunk_len); + // Handle remainder bits if any + if right_chunks.remainder_len() > 0 { + handle_mutable_buffer_remainder( + &mut op, + remainder_bytes, + right_chunks.remainder_bits(), + right_chunks.remainder_len(), + ) + } +} - &mut left.as_slice_mut()[byte_offset..] - }; - // cast to *const u64 should be fine since we are using read_unaligned below - #[allow(clippy::cast_ptr_alignment)] - let mut left_buffer_mut_u64_ptr = left_buffer_mut.as_mut_ptr() as *mut u64; +/// Centralized structure to handle a mutable u8 slice as a mutable u64 pointer. +/// +/// Handle the following: +/// 1. the lifetime is correct +/// 2. we read/write within the bounds +/// 3. We read and write using unaligned +/// +/// This does not deallocate the underlying pointer when dropped +/// +/// This is the only place that uses unsafe code to read and write unaligned +/// +struct U64UnalignedSlice<'a> { + /// Pointer to the start of the u64 data + /// + /// We are using raw pointer as the data came from a u8 slice so we need to read and write unaligned + ptr: *mut u64, - let mut right_chunks_iter = right_chunks.iter(); + /// Number of u64 elements + len: usize, - // If not only remainder bytes - let had_any_chunks = right_chunks_iter.len() > 0; + /// Marker to tie the lifetime of the pointer to the lifetime of the u8 slice + _marker: std::marker::PhantomData<&'a u8>, +} - // 2. Process complete u64 chunks - { - // Process the first chunk outside the loop to avoid incrementing - // the pointer after the last read - if let Some(right) = right_chunks_iter.next() { +impl<'a> U64UnalignedSlice<'a> { + /// Create a new [`U64UnalignedSlice`] from a [`MutableBuffer`] + /// + /// return the [`U64UnalignedSlice`] and slice of bytes that are not part of the u64 chunks (guaranteed to be less than 8 bytes) + /// + fn split( + mutable_buffer: &'a mut MutableBuffer, + offset_in_bits: usize, + len_in_bits: usize, + ) -> (Self, &'a mut [u8]) { + // 1. Prepare the buffers + let left_buffer_mut: &mut [u8] = { + let last_offset = bit_util::ceil(offset_in_bits + len_in_bits, 8); + assert!(last_offset <= mutable_buffer.len()); + + let byte_offset = offset_in_bits / 8; + + &mut mutable_buffer.as_slice_mut()[byte_offset..last_offset] + }; + + const U64_SIZE_IN_BITS: usize = size_of::() * 8; + let number_of_u64_we_can_fit = len_in_bits / U64_SIZE_IN_BITS; + + // 2. Split + let u64_len_in_bytes = number_of_u64_we_can_fit * size_of::(); + + assert!(u64_len_in_bytes <= left_buffer_mut.len()); + let (bytes_for_u64, remainder) = left_buffer_mut.split_at_mut( + u64_len_in_bytes + ); + + let ptr = bytes_for_u64.as_mut_ptr() as *mut u64; + + let this = Self { + ptr, + len: number_of_u64_we_can_fit, + _marker: std::marker::PhantomData, + }; + + (this, remainder) + } + + + fn len(&self) -> usize { + self.len + } + + /// Modify the underlying u64 data in place using a binary operation + /// with another iterator. + fn zip_modify( + mut self, + mut zip_iter: impl ExactSizeIterator, + mut map: impl FnMut(u64, u64) -> u64, + ) { + assert_eq!(self.len, zip_iter.len()); + + // In order to avoid advancing the pointer at the end of the loop which will + // make the last pointer invalid, we handle the first element outside the loop + // and then advance the pointer at the start of the loop + // making sure that the iterator is not empty + if let Some(right) = zip_iter.next() { + // SAFETY: We asserted that the iterator length and the current length are the same + // and the iterator is not empty, so the pointer is valid unsafe { - run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + self.modify_self(right, &mut map); } + + // Because this consumes self we don't update the length } - for right in right_chunks_iter { - // Increase the pointer for the next iteration - // we are increasing the pointer before reading because we already read the first chunk above - left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; + for right in zip_iter { + // Advance the pointer + // + // SAFETY: We asserted that the iterator length and the current length are the same + self.ptr = unsafe { self.ptr.add(1) }; + // SAFETY: the pointer is valid as we are within the length unsafe { - run_op_on_mutable_pointer_and_single_value(&mut op, left_buffer_mut_u64_ptr, right); + self.modify_self(right, &mut map); } + + // Because this consumes self we don't update the length } } - // Handle remainder bits if any - if right_chunks.remainder_len() > 0 { - { - // If we had any chunks we only advance the pointer at the start. - // so we need to advance it again if we have a remainder - let advance_pointer_count = if had_any_chunks { 1 } else { 0 }; - left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(advance_pointer_count) } + /// Centralized function to correctly read the current u64 value and write back the result + /// + /// # SAFETY + /// the caller must ensure that the pointer is valid for reads and writes + /// + #[inline] + unsafe fn modify_self(&mut self, right: u64, mut map: impl FnMut(u64, u64) -> u64) { + // Safety the caller must ensure pointer point to a valid u64 + let current_input = unsafe { + self.ptr + // Reading unaligned as we came from u8 slice + .read_unaligned() + // bit-packed buffers are stored starting with the least-significant byte first + // so when reading as u64 on a big-endian machine, the bytes need to be swapped + .to_le() + }; + + let combined = map(current_input, right); + + // Write the result back + // + // The pointer came from mutable u8 slice so the pointer is valid for writes, + // and we need to write unaligned + unsafe { self.ptr.write_unaligned(combined) } + } + + /// Modify the underlying u64 data in place using a unary operation. + fn modify(mut self, mut map: impl FnMut(u64) -> u64) { + if self.len == 0 { + return; } - let left_buffer_mut_u8_ptr = left_buffer_mut_u64_ptr as *mut u8; - handle_mutable_buffer_remainder( - &mut op, - left_buffer_mut_u8_ptr, - right_chunks.remainder_bits(), - right_chunks.remainder_len(), - ) + // In order to avoid advancing the pointer at the end of the loop which will + // make the last pointer invalid, we handle the first element outside the loop + // and then advance the pointer at the start of the loop + // making sure that the iterator is not empty + unsafe { + // I hope the function get inlined and the compiler remove the dead right parameter + self.modify_self(0, &mut |left, _| map(left)); + + // Because this consumes self we don't update the length + } + + for _ in 1..self.len { + // Advance the pointer + // + // SAFETY: we only advance the pointer within the length and not beyond + self.ptr = unsafe { self.ptr.add(1) }; + + // SAFETY: the pointer is valid as we are within the length + unsafe { + // I hope the function get inlined and the compiler remove the dead right parameter + self.modify_self(0, &mut |left, _| map(left)); + } + + // Because this consumes self we don't update the length + } } } @@ -398,20 +527,20 @@ fn mutable_buffer_byte_aligned_bitwise_bin_op_helper( #[inline] fn handle_mutable_buffer_remainder( op: &mut F, - start_remainder_mut_ptr: *mut u8, + start_remainder_mut_slice: &mut [u8], right_remainder_bits: u64, remainder_len: usize, ) where F: FnMut(u64, u64) -> u64, { // Only read from mut pointer the number of remainder bits - let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); + let left_remainder_bits = get_remainder_bits(start_remainder_mut_slice, remainder_len); // Apply the operation let rem = op(left_remainder_bits, right_remainder_bits); // Write only the relevant bits back the result to the mutable pointer - set_remainder_bits(start_remainder_mut_ptr, rem, remainder_len); + set_remainder_bits(start_remainder_mut_slice, rem, remainder_len); } /// Write remainder bits back to buffer while preserving bits outside the range. @@ -421,11 +550,13 @@ fn handle_mutable_buffer_remainder( /// /// # Arguments /// -/// * `start_remainder_mut_ptr` - Pointer to the start of remainder bytes +/// * `start_remainder_mut_slice` - the slice of bytes to write the remainder bits to /// * `rem` - The result bits to write /// * `remainder_len` - Number of bits to write #[inline] -fn set_remainder_bits(start_remainder_mut_ptr: *mut u8, rem: u64, remainder_len: usize) { +fn set_remainder_bits(start_remainder_mut_slice: &mut [u8], rem: u64, remainder_len: usize) { + assert_ne!(start_remainder_mut_slice.len(), 0, "start_remainder_mut_slice must not be empty"); + assert!(remainder_len < 64, "remainder_len must be less than 64"); // Need to update the remainder bytes in the mutable buffer // but not override the bits outside the remainder @@ -433,12 +564,11 @@ fn set_remainder_bits(start_remainder_mut_ptr: *mut u8, rem: u64, remainder_len: // to preserve the bits outside the remainder let rem = { // 1. Read the byte that we will override - let current = { - let last_byte_position = remainder_len / 8; - let last_byte_ptr = unsafe { start_remainder_mut_ptr.add(last_byte_position) }; + let current = start_remainder_mut_slice.last() + // Unwrap as we already validated the slice is not empty + .unwrap(); - unsafe { std::ptr::read(last_byte_ptr) as u64 } - }; + let current = *current as u64; // Mask where the bits that are inside the remainder are 1 // and the bits outside the remainder are 0 @@ -459,16 +589,16 @@ fn set_remainder_bits(start_remainder_mut_ptr: *mut u8, rem: u64, remainder_len: // Write back the result to the mutable pointer { - let remainder_bytes = ceil(remainder_len, 8); + let remainder_bytes = bit_util::ceil(remainder_len, 8); // we are counting starting from the least significant bit, so to_le_bytes should be correct let rem = &rem.to_le_bytes()[0..remainder_bytes]; // this assumes that `[ToByteSlice]` can be copied directly // without calling `to_byte_slice` for each element, - // which is correct for all ArrowNativeType implementations. + // which is correct for all ArrowNativeType implementations including u64. let src = rem.as_ptr(); - unsafe { std::ptr::copy_nonoverlapping(src, start_remainder_mut_ptr, remainder_bytes) }; + unsafe { std::ptr::copy_nonoverlapping(src, start_remainder_mut_slice.as_mut_ptr(), remainder_bytes) }; } } @@ -485,56 +615,15 @@ fn set_remainder_bits(start_remainder_mut_ptr: *mut u8, rem: u64, remainder_len: /// /// A u64 containing the bits in the least significant positions #[inline] -fn get_remainder_bits(remainder_ptr: *const u8, remainder_len: usize) -> u64 { - let bit_len = remainder_len; - // number of bytes to read - let byte_len = ceil(bit_len, 8); - // pointer to remainder bytes after all complete chunks - let base = remainder_ptr; - - let mut bits = unsafe { std::ptr::read(base) } as u64; - for i in 1..byte_len { - let byte = unsafe { std::ptr::read(base.add(i)) }; - bits |= (byte as u64) << (i * 8); - } +fn get_remainder_bits(remainder: &[u8], remainder_len: usize) -> u64 { + assert!(remainder.len() < 64, "remainder_len must be less than 64"); + assert_eq!(remainder.len(), bit_util::ceil(remainder_len, 8), "remainder and remainder len ceil must be the same"); - bits & ((1 << bit_len) - 1) -} + let bits = remainder.iter().enumerate().fold(0_u64, |acc, (index, &byte)| { + acc | (byte as u64) << (index * 8) + }); -/// Apply a binary operation to a u64 in memory. -/// -/// Reads a u64 from the pointer, applies the operation with the right operand, -/// and writes the result back. Handles endianness correctly for bit-packed buffers. -/// -/// # Safety -/// -/// The pointer must be valid for reads and writes of u64 values. -/// Unaligned access is handled correctly via `read_unaligned` and `write_unaligned`. -/// -/// # Arguments -/// -/// * `op` - Binary operation to apply -/// * `left_buffer_mut_ptr` - Pointer to the left operand u64 -/// * `right` - Right operand value -#[inline] -unsafe fn run_op_on_mutable_pointer_and_single_value( - op: &mut F, - left_buffer_mut_ptr: *mut u64, - right: u64, -) where - F: FnMut(u64, u64) -> u64, -{ - // 1. Read the current value from the mutable buffer - // - // bit-packed buffers are stored starting with the least-significant byte first - // so when reading as u64 on a big-endian machine, the bytes need to be swapped - let current = unsafe { std::ptr::read_unaligned(left_buffer_mut_ptr).to_le() }; - - // 2. Get the new value by applying the operation - let combined = op(current, right); - - // 3. Write the new value back to the mutable buffer - unsafe { std::ptr::write_unaligned(left_buffer_mut_ptr, combined) }; + bits & ((1 << remainder_len) - 1) } /// Perform bitwise unary operation on byte-aligned buffer. @@ -564,84 +653,22 @@ fn mutable_byte_aligned_bitwise_unary_op_helper( "left_offset_in_bits must be byte aligned" ); - let number_of_u64_chunks = len_in_bits / 64; let remainder_len = len_in_bits % 64; - let left_buffer_mut: &mut [u8] = { - assert!(ceil(left_offset_in_bits + len_in_bits, 8) <= left.len()); - - let byte_offset = left_offset_in_bits / 8; + let (complete_u64_chunks, remainder_bytes) = + U64UnalignedSlice::split(left, left_offset_in_bits, len_in_bits); - &mut left.as_slice_mut()[byte_offset..] - }; - - // cast to *const u64 should be fine since we are using read_unaligned below - #[allow(clippy::cast_ptr_alignment)] - let mut left_buffer_mut_u64_ptr = left_buffer_mut.as_mut_ptr() as *mut u64; - - // If not only remainder bytes - let had_any_chunks = number_of_u64_chunks > 0; - - // Process the first chunk outside the loop - if had_any_chunks { - unsafe { - run_op_on_mutable_pointer(&mut op, left_buffer_mut_u64_ptr); - } - } - - for _ in 1..number_of_u64_chunks { - // Increase the pointer for the next iteration - left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(1) }; + assert_eq!(bit_util::ceil(remainder_len, 8), remainder_bytes.len()); - unsafe { - run_op_on_mutable_pointer(&mut op, left_buffer_mut_u64_ptr); - } - } + // 2. Process complete u64 chunks + complete_u64_chunks.modify(&mut op); // Handle remainder bits if any if remainder_len > 0 { - { - // If we had any chunks we only advance the pointer at the start, - // so we need to advance it again if we have a remainder - let advance_pointer_count = if had_any_chunks { 1 } else { 0 }; - left_buffer_mut_u64_ptr = unsafe { left_buffer_mut_u64_ptr.add(advance_pointer_count) } - } - let left_buffer_mut_u8_ptr = left_buffer_mut_u64_ptr as *mut u8; - - handle_mutable_buffer_remainder_unary(&mut op, left_buffer_mut_u8_ptr, remainder_len); + handle_mutable_buffer_remainder_unary(&mut op, remainder_bytes, remainder_len) } } -/// Apply a unary operation to a u64 in memory. -/// -/// Reads a u64 from the pointer, applies the operation, and writes the result back. -/// -/// # Safety -/// -/// The pointer must be valid for reads and writes of u64 values. -/// -/// # Arguments -/// -/// * `op` - Unary operation to apply -/// * `buffer_mut_ptr` - Pointer to the u64 -#[inline] -unsafe fn run_op_on_mutable_pointer(op: &mut F, buffer_mut_ptr: *mut u64) -where - F: FnMut(u64) -> u64, -{ - // 1. Read the current value from the mutable buffer - // - // bit-packed buffers are stored starting with the least-significant byte first - // so when reading as u64 on a big-endian machine, the bytes need to be swapped - let current = unsafe { std::ptr::read_unaligned(buffer_mut_ptr).to_le() }; - - // 2. Get the new value by applying the operation - let result = op(current); - - // 3. Write the new value back to the mutable buffer - unsafe { std::ptr::write_unaligned(buffer_mut_ptr, result) }; -} - /// Handle remainder bits (< 64 bits) for unary operations. /// /// This function processes the bits that don't form a complete u64 chunk, @@ -650,24 +677,24 @@ where /// # Arguments /// /// * `op` - Unary operation to apply -/// * `start_remainder_mut_ptr` - Pointer to the start of remainder bytes +/// * `start_remainder_mut` - Slice of bytes to write the remainder bits to /// * `remainder_len` - Number of remainder bits #[inline] fn handle_mutable_buffer_remainder_unary( op: &mut F, - start_remainder_mut_ptr: *mut u8, + start_remainder_mut: &mut [u8], remainder_len: usize, ) where F: FnMut(u64) -> u64, { // Only read from mut pointer the number of remainder bits - let left_remainder_bits = get_remainder_bits(start_remainder_mut_ptr, remainder_len); + let left_remainder_bits = get_remainder_bits(start_remainder_mut, remainder_len); // Apply the operation let rem = op(left_remainder_bits); // Write only the relevant bits back the result to the mutable pointer - set_remainder_bits(start_remainder_mut_ptr, rem, remainder_len); + set_remainder_bits(start_remainder_mut, rem, remainder_len); } /// Apply a bitwise operation to a mutable buffer and update it in-place. From 6b7bfe92e2743debd70df7dc2a8c45d5a918fe1a Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Wed, 15 Oct 2025 16:06:16 +0300 Subject: [PATCH 12/15] format --- arrow-buffer/src/buffer/mutable_ops.rs | 72 ++++++++++++++++---------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index 89b22579daf7..0784672bedaf 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -273,16 +273,21 @@ fn read_up_to_byte_from_offset( let number_of_bytes_to_read = bit_util::ceil(number_of_bits_to_read + bit_offset, 8); - // number of bytes to read - // might be one more than sizeof(u64) if the offset is in the middle of a byte - assert!(slice.len() >= number_of_bytes_to_read); - - let mut bits = slice[0] >> bit_offset; - for (i, &byte) in slice.iter().take(number_of_bytes_to_read).enumerate().skip(1) { - bits |= byte << (i * 8 - bit_offset); - } + // number of bytes to read + // might be one more than sizeof(u64) if the offset is in the middle of a byte + assert!(slice.len() >= number_of_bytes_to_read); + + let mut bits = slice[0] >> bit_offset; + for (i, &byte) in slice + .iter() + .take(number_of_bytes_to_read) + .enumerate() + .skip(1) + { + bits |= byte << (i * 8 - bit_offset); + } - bits & ((1 << number_of_bits_to_read) - 1) + bits & ((1 << number_of_bits_to_read) - 1) } /// Perform bitwise binary operation on byte-aligned buffers (i.e. not offsetting into a middle of a byte). @@ -318,7 +323,7 @@ fn mutable_buffer_byte_aligned_bitwise_bin_op_helper( // 1. Prepare the buffers let (complete_u64_chunks, remainder_bytes) = - U64UnalignedSlice::split(left, left_offset_in_bits, len_in_bits); + U64UnalignedSlice::split(left, left_offset_in_bits, len_in_bits); let right_chunks = BitChunks::new(right.as_slice(), right_offset_in_bits, len_in_bits); assert_eq!( @@ -343,7 +348,6 @@ fn mutable_buffer_byte_aligned_bitwise_bin_op_helper( } } - /// Centralized structure to handle a mutable u8 slice as a mutable u64 pointer. /// /// Handle the following: @@ -388,16 +392,13 @@ impl<'a> U64UnalignedSlice<'a> { &mut mutable_buffer.as_slice_mut()[byte_offset..last_offset] }; - const U64_SIZE_IN_BITS: usize = size_of::() * 8; - let number_of_u64_we_can_fit = len_in_bits / U64_SIZE_IN_BITS; + let number_of_u64_we_can_fit = len_in_bits / (u64::BITS as usize); // 2. Split let u64_len_in_bytes = number_of_u64_we_can_fit * size_of::(); assert!(u64_len_in_bytes <= left_buffer_mut.len()); - let (bytes_for_u64, remainder) = left_buffer_mut.split_at_mut( - u64_len_in_bytes - ); + let (bytes_for_u64, remainder) = left_buffer_mut.split_at_mut(u64_len_in_bytes); let ptr = bytes_for_u64.as_mut_ptr() as *mut u64; @@ -410,7 +411,6 @@ impl<'a> U64UnalignedSlice<'a> { (this, remainder) } - fn len(&self) -> usize { self.len } @@ -555,7 +555,11 @@ fn handle_mutable_buffer_remainder( /// * `remainder_len` - Number of bits to write #[inline] fn set_remainder_bits(start_remainder_mut_slice: &mut [u8], rem: u64, remainder_len: usize) { - assert_ne!(start_remainder_mut_slice.len(), 0, "start_remainder_mut_slice must not be empty"); + assert_ne!( + start_remainder_mut_slice.len(), + 0, + "start_remainder_mut_slice must not be empty" + ); assert!(remainder_len < 64, "remainder_len must be less than 64"); // Need to update the remainder bytes in the mutable buffer // but not override the bits outside the remainder @@ -564,9 +568,10 @@ fn set_remainder_bits(start_remainder_mut_slice: &mut [u8], rem: u64, remainder_ // to preserve the bits outside the remainder let rem = { // 1. Read the byte that we will override - let current = start_remainder_mut_slice.last() - // Unwrap as we already validated the slice is not empty - .unwrap(); + let current = start_remainder_mut_slice + .last() + // Unwrap as we already validated the slice is not empty + .unwrap(); let current = *current as u64; @@ -598,7 +603,13 @@ fn set_remainder_bits(start_remainder_mut_slice: &mut [u8], rem: u64, remainder_ // without calling `to_byte_slice` for each element, // which is correct for all ArrowNativeType implementations including u64. let src = rem.as_ptr(); - unsafe { std::ptr::copy_nonoverlapping(src, start_remainder_mut_slice.as_mut_ptr(), remainder_bytes) }; + unsafe { + std::ptr::copy_nonoverlapping( + src, + start_remainder_mut_slice.as_mut_ptr(), + remainder_bytes, + ) + }; } } @@ -617,11 +628,18 @@ fn set_remainder_bits(start_remainder_mut_slice: &mut [u8], rem: u64, remainder_ #[inline] fn get_remainder_bits(remainder: &[u8], remainder_len: usize) -> u64 { assert!(remainder.len() < 64, "remainder_len must be less than 64"); - assert_eq!(remainder.len(), bit_util::ceil(remainder_len, 8), "remainder and remainder len ceil must be the same"); + assert_eq!( + remainder.len(), + bit_util::ceil(remainder_len, 8), + "remainder and remainder len ceil must be the same" + ); - let bits = remainder.iter().enumerate().fold(0_u64, |acc, (index, &byte)| { - acc | (byte as u64) << (index * 8) - }); + let bits = remainder + .iter() + .enumerate() + .fold(0_u64, |acc, (index, &byte)| { + acc | (byte as u64) << (index * 8) + }); bits & ((1 << remainder_len) - 1) } @@ -656,7 +674,7 @@ fn mutable_byte_aligned_bitwise_unary_op_helper( let remainder_len = len_in_bits % 64; let (complete_u64_chunks, remainder_bytes) = - U64UnalignedSlice::split(left, left_offset_in_bits, len_in_bits); + U64UnalignedSlice::split(left, left_offset_in_bits, len_in_bits); assert_eq!(bit_util::ceil(remainder_len, 8), remainder_bytes.len()); From aec92d61ed5f5683a893202d8b0be40b11ad88a7 Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Sun, 26 Oct 2025 16:09:35 +0200 Subject: [PATCH 13/15] add reproduction test --- arrow-buffer/src/buffer/mutable_ops.rs | 30 ++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index 0784672bedaf..95b05889b8fd 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -140,7 +140,10 @@ pub fn mutable_bitwise_bin_op_helper( ); } else { // If we are not byte aligned, run `op` on the first few bits to reach byte alignment - let bits_to_next_byte = 8 - left_bit_offset; + let bits_to_next_byte = (8 - left_bit_offset) + // Minimum with the amount of bits we need to process + // to avoid reading out of bounds + .min(len_in_bits); { let right_byte_offset = right_offset_in_bits / 8; @@ -275,7 +278,7 @@ fn read_up_to_byte_from_offset( // number of bytes to read // might be one more than sizeof(u64) if the offset is in the middle of a byte - assert!(slice.len() >= number_of_bytes_to_read); + assert!(slice.len() >= number_of_bytes_to_read, "slice is too small"); let mut bits = slice[0] >> bit_offset; for (i, &byte) in slice @@ -763,11 +766,11 @@ pub fn mutable_bitwise_unary_op_helper( op, ); } else { + align_to_byte(&mut op, mutable_buffer, offset_in_bits); + // If we are not byte aligned we will read the first few bits let bits_to_next_byte = 8 - left_bit_offset; - align_to_byte(&mut op, mutable_buffer, offset_in_bits); - let offset_in_bits = offset_in_bits + bits_to_next_byte; let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); @@ -1253,4 +1256,23 @@ mod tests { let data = vec![true, false, true, false]; test_mutable_buffer_unary_op_helper(&data, 0, 0, |a| !a, |a| !a); } + + #[test] + fn test_less_than_byte_unaligned_and_not_enough_bits() { + let left_offset_in_bits = 2; + let right_offset_in_bits = 4; + let len_in_bits = 1; + + // Single byte + let right = (0..8).map(|i| (i / 2) % 2 == 0).collect::>(); + // less than a byte + let left = (0..3).map(|i| i % 2 == 0).collect::>(); + test_all_binary_ops( + &left, + &right, + left_offset_in_bits, + right_offset_in_bits, + len_in_bits, + ); + } } From db3e853bde8b0097db5d7598c2a749483de72ffd Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Sun, 26 Oct 2025 22:44:36 +0200 Subject: [PATCH 14/15] extract, cleanup and add comments --- arrow-buffer/src/buffer/immutable.rs | 6 + arrow-buffer/src/buffer/mod.rs | 1 - arrow-buffer/src/buffer/mutable.rs | 6 + arrow-buffer/src/buffer/mutable_ops.rs | 827 +++++++++---------------- arrow-buffer/src/builder/boolean.rs | 43 +- arrow-buffer/src/util/bit_util.rs | 122 ++++ 6 files changed, 448 insertions(+), 557 deletions(-) diff --git a/arrow-buffer/src/buffer/immutable.rs b/arrow-buffer/src/buffer/immutable.rs index 20eb966a8f08..2ccbe0f6ecfb 100644 --- a/arrow-buffer/src/buffer/immutable.rs +++ b/arrow-buffer/src/buffer/immutable.rs @@ -524,6 +524,12 @@ impl std::ops::Deref for Buffer { } } +impl AsRef<[u8]> for &Buffer { + fn as_ref(&self) -> &[u8] { + self.as_slice() + } +} + impl From for Buffer { #[inline] fn from(buffer: MutableBuffer) -> Self { diff --git a/arrow-buffer/src/buffer/mod.rs b/arrow-buffer/src/buffer/mod.rs index 676d64152e47..11275df77e4d 100644 --- a/arrow-buffer/src/buffer/mod.rs +++ b/arrow-buffer/src/buffer/mod.rs @@ -26,7 +26,6 @@ pub use mutable::*; mod ops; pub use ops::*; mod mutable_ops; -pub use mutable_ops::*; mod scalar; pub use scalar::*; mod boolean; diff --git a/arrow-buffer/src/buffer/mutable.rs b/arrow-buffer/src/buffer/mutable.rs index 93d9d6b9ad84..92282d26001c 100644 --- a/arrow-buffer/src/buffer/mutable.rs +++ b/arrow-buffer/src/buffer/mutable.rs @@ -743,6 +743,12 @@ impl std::ops::DerefMut for MutableBuffer { } } +impl AsRef<[u8]> for &MutableBuffer { + fn as_ref(&self) -> &[u8] { + self.as_slice() + } +} + impl Drop for MutableBuffer { fn drop(&mut self) { if self.layout.size() != 0 { diff --git a/arrow-buffer/src/buffer/mutable_ops.rs b/arrow-buffer/src/buffer/mutable_ops.rs index 95b05889b8fd..7a9ae25d90d7 100644 --- a/arrow-buffer/src/buffer/mutable_ops.rs +++ b/arrow-buffer/src/buffer/mutable_ops.rs @@ -15,282 +15,194 @@ // specific language governing permissions and limitations // under the License. -use super::{Buffer, MutableBuffer}; -use crate::BooleanBufferBuilder; +use super::MutableBuffer; use crate::bit_chunk_iterator::BitChunks; use crate::util::bit_util; -/// What can be used as the right-hand side (RHS) buffer in mutable operations. -/// -/// this is not mutated. -/// -/// # Implementation notes -/// -/// ## Why `pub(crate)`? -/// This is because we don't want this trait to expose the inner buffer to the public. -/// this is the trait implementor choice. -/// -pub(crate) trait BufferSupportedRhs { - fn as_slice(&self) -> &[u8]; -} - -impl BufferSupportedRhs for Buffer { - fn as_slice(&self) -> &[u8] { - self.as_slice() - } -} - -impl BufferSupportedRhs for MutableBuffer { - fn as_slice(&self) -> &[u8] { - self.as_slice() - } -} - -impl BufferSupportedRhs for BooleanBufferBuilder { - fn as_slice(&self) -> &[u8] { - self.as_slice() - } -} - -/// Trait that will be operated on as the left-hand side (LHS) buffer in mutable operations. -/// -/// This consumer of the trait must satisfies the following guarantees: -/// 1. It will not change the length of the buffer. -/// -/// # Implementation notes -/// -/// ## Why is this trait `pub(crate)`? -/// Because we don't wanna expose the inner mutable buffer to the public. -/// as this is the choice of the implementor of the trait and sometimes it is not desirable -/// (e.g. `BooleanBufferBuilder`). -/// -/// ## Why this trait is needed, can't we just use `MutableBuffer` directly? -/// Sometimes we don't want to expose the inner `MutableBuffer` -/// so it can't be misused. -/// -/// For example, [`BooleanBufferBuilder`] does not expose the inner `MutableBuffer` -/// as exposing it will allow the user to change the length of the buffer that will make the -/// `BooleanBufferBuilder` invalid. -/// -pub(crate) trait MutableOpsBufferSupportedLhs { - /// Get a mutable reference to the inner `MutableBuffer`. +impl MutableBuffer { + /// Apply a binary bitwise operation on self (mutate) with respect to another buffer (right). /// - /// This is used to perform in-place operations on the buffer. + /// # Arguments /// - /// the caller must ensure that the length of the buffer is not changed. - fn inner_mutable_buffer(&mut self) -> &mut MutableBuffer; -} - -impl MutableOpsBufferSupportedLhs for MutableBuffer { - fn inner_mutable_buffer(&mut self) -> &mut MutableBuffer { - self - } -} - -/// Apply a binary bitwise operation to two bit-packed buffers. -/// -/// This is the main entry point for binary operations. It handles both byte-aligned -/// and non-byte-aligned cases by delegating to specialized helper functions. -/// -/// # Arguments -/// -/// * `left` - The left mutable buffer to be modified in-place -/// * `left_offset_in_bits` - Starting bit offset in the left buffer -/// * `right` - The right buffer (as byte slice) -/// * `right_offset_in_bits` - Starting bit offset in the right buffer -/// * `len_in_bits` - Number of bits to process -/// * `op` - Binary operation to apply (e.g., `|a, b| a & b`) -/// -#[allow( - private_bounds, - reason = "MutableOpsBufferSupportedLhs and BufferSupportedRhs exposes the inner internals which is the implementor choice and we dont want to leak internals" -)] -pub fn mutable_bitwise_bin_op_helper( - left: &mut impl MutableOpsBufferSupportedLhs, - left_offset_in_bits: usize, - right: &impl BufferSupportedRhs, - right_offset_in_bits: usize, - len_in_bits: usize, - mut op: F, -) where - F: FnMut(u64, u64) -> u64, -{ - if len_in_bits == 0 { - return; - } - - let mutable_buffer = left.inner_mutable_buffer(); + /// * `self` - The mutable buffer to be modified in-place + /// * `offset_in_bits` - Starting bit offset in Self buffer + /// * `right` - slice of bit-packed bytes in LSB order + /// * `right_offset_in_bits` - Starting bit offset in the right buffer + /// * `len_in_bits` - Number of bits to process + /// * `op` - Binary operation to apply (e.g., `|a, b| a & b`) + /// + pub fn bitwise_binary_op( + &mut self, + offset_in_bits: usize, + right: impl AsRef<[u8]>, + right_offset_in_bits: usize, + len_in_bits: usize, + mut op: F, + ) where + F: FnMut(u64, u64) -> u64, + { + if len_in_bits == 0 { + return; + } - let mutable_buffer_len = mutable_buffer.len(); - let mutable_buffer_cap = mutable_buffer.capacity(); + let mutable_buffer_len = self.len(); + let mutable_buffer_cap = self.capacity(); - // offset inside a byte - let left_bit_offset = left_offset_in_bits % 8; + // offset inside a byte + let bit_offset = offset_in_bits % 8; - let is_mutable_buffer_byte_aligned = left_bit_offset == 0; + let is_mutable_buffer_byte_aligned = bit_offset == 0; - if is_mutable_buffer_byte_aligned { - mutable_buffer_byte_aligned_bitwise_bin_op_helper( - mutable_buffer, - left_offset_in_bits, - right, - right_offset_in_bits, - len_in_bits, - op, - ); - } else { - // If we are not byte aligned, run `op` on the first few bits to reach byte alignment - let bits_to_next_byte = (8 - left_bit_offset) - // Minimum with the amount of bits we need to process - // to avoid reading out of bounds - .min(len_in_bits); - - { - let right_byte_offset = right_offset_in_bits / 8; - - // Read the same amount of bits from the right buffer - let right_first_byte: u8 = read_up_to_byte_from_offset( - &right.as_slice()[right_byte_offset..], - bits_to_next_byte, - // Right bit offset - right_offset_in_bits % 8, + if is_mutable_buffer_byte_aligned { + byte_aligned_bitwise_bin_op_helper( + self, + offset_in_bits, + right, + right_offset_in_bits, + len_in_bits, + op, ); + } else { + // If we are not byte aligned, run `op` on the first few bits to reach byte alignment + let bits_to_next_byte = (8 - bit_offset) + // Minimum with the amount of bits we need to process + // to avoid reading out of bounds + .min(len_in_bits); + + { + let right_byte_offset = right_offset_in_bits / 8; + + // Read the same amount of bits from the right buffer + let right_first_byte: u8 = crate::util::bit_util::read_up_to_byte_from_offset( + &right.as_ref()[right_byte_offset..], + bits_to_next_byte, + // Right bit offset + right_offset_in_bits % 8, + ); + + align_to_byte( + self, + // Hope it gets inlined + &mut |left| op(left, right_first_byte as u64), + offset_in_bits, + ); + } + + let offset_in_bits = offset_in_bits + bits_to_next_byte; + let right_offset_in_bits = right_offset_in_bits + bits_to_next_byte; + let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); + + if len_in_bits == 0 { + // Making sure that our guarantee that the length and capacity of the mutable buffer + // will not change is upheld + assert_eq!( + self.len(), + mutable_buffer_len, + "The length of the mutable buffer must not change" + ); + assert_eq!( + self.capacity(), + mutable_buffer_cap, + "The capacity of the mutable buffer must not change" + ); + + return; + } - align_to_byte( - // Hope it gets inlined - &mut |left| op(left, right_first_byte as u64), - mutable_buffer, - left_offset_in_bits, + // We are now byte aligned + byte_aligned_bitwise_bin_op_helper( + self, + offset_in_bits, + right, + right_offset_in_bits, + len_in_bits, + op, ); } - let left_offset_in_bits = left_offset_in_bits + bits_to_next_byte; - let right_offset_in_bits = right_offset_in_bits + bits_to_next_byte; - let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); + // Making sure that our guarantee that the length and capacity of the mutable buffer + // will not change is upheld + assert_eq!( + self.len(), + mutable_buffer_len, + "The length of the mutable buffer must not change" + ); + assert_eq!( + self.capacity(), + mutable_buffer_cap, + "The capacity of the mutable buffer must not change" + ); + } + /// Apply a bitwise operation to a mutable buffer and update it in-place. + /// + /// # Arguments + /// + /// * `offset_in_bits` - Starting bit offset for the current buffer + /// * `len_in_bits` - Number of bits to process + /// * `op` - Unary operation to apply (e.g., `|a| !a`) + /// + pub fn bitwise_unary_op(&mut self, offset_in_bits: usize, len_in_bits: usize, mut op: F) + where + F: FnMut(u64) -> u64, + { if len_in_bits == 0 { - // Making sure that our guarantee that the length and capacity of the mutable buffer - // will not change is upheld - assert_eq!( - mutable_buffer.len(), - mutable_buffer_len, - "The length of the mutable buffer must not change" - ); - assert_eq!( - mutable_buffer.capacity(), - mutable_buffer_cap, - "The capacity of the mutable buffer must not change" - ); - return; } - // We are now byte aligned - mutable_buffer_byte_aligned_bitwise_bin_op_helper( - mutable_buffer, - left_offset_in_bits, - right, - right_offset_in_bits, - len_in_bits, - op, - ); - } + let mutable_buffer_len = self.len(); + let mutable_buffer_cap = self.capacity(); - // Making sure that our guarantee that the length and capacity of the mutable buffer - // will not change is upheld - assert_eq!( - mutable_buffer.len(), - mutable_buffer_len, - "The length of the mutable buffer must not change" - ); - assert_eq!( - mutable_buffer.capacity(), - mutable_buffer_cap, - "The capacity of the mutable buffer must not change" - ); -} + // offset inside a byte + let left_bit_offset = offset_in_bits % 8; -/// Align to byte boundary by applying operation to bits before the next byte boundary. -/// -/// This function handles non-byte-aligned operations by processing bits from the current -/// position up to the next byte boundary, while preserving all other bits in the byte. -/// -/// # Arguments -/// -/// * `op` - Unary operation to apply -/// * `buffer` - The mutable buffer to modify -/// * `offset_in_bits` - Starting bit offset (not byte-aligned) -fn align_to_byte(op: &mut F, buffer: &mut MutableBuffer, offset_in_bits: usize) -where - F: FnMut(u64) -> u64, -{ - let byte_offset = offset_in_bits / 8; - let bit_offset = offset_in_bits % 8; + let is_mutable_buffer_byte_aligned = left_bit_offset == 0; - // 1. read the first byte from the buffer - let first_byte: u8 = buffer.as_slice()[byte_offset]; + if is_mutable_buffer_byte_aligned { + byte_aligned_bitwise_unary_op_helper(self, offset_in_bits, len_in_bits, op); + } else { + align_to_byte(self, &mut op, offset_in_bits); - // 2. Shift byte by the bit offset, keeping only the relevant bits - let relevant_first_byte = first_byte >> bit_offset; + // If we are not byte aligned we will read the first few bits + let bits_to_next_byte = 8 - left_bit_offset; - // 3. run the op on the first byte only - let result_first_byte = op(relevant_first_byte as u64) as u8; + let offset_in_bits = offset_in_bits + bits_to_next_byte; + let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); - // 4. Shift back the result to the original position - let result_first_byte = result_first_byte << bit_offset; + if len_in_bits == 0 { + // Making sure that our guarantee that the length and capacity of the mutable buffer + // will not change is upheld + assert_eq!( + self.len(), + mutable_buffer_len, + "The length of the mutable buffer must not change" + ); + assert_eq!( + self.capacity(), + mutable_buffer_cap, + "The capacity of the mutable buffer must not change" + ); - // 5. Mask the bits that are outside the relevant bits in the byte - // so the bits until bit_offset are 1 and the rest are 0 - let mask_for_first_bit_offset = (1 << bit_offset) - 1; - - let result_first_byte = - (first_byte & mask_for_first_bit_offset) | (result_first_byte & !mask_for_first_bit_offset); + return; + } - // 6. write back the result to the buffer - buffer.as_slice_mut()[byte_offset] = result_first_byte; -} + // We are now byte aligned + byte_aligned_bitwise_unary_op_helper(self, offset_in_bits, len_in_bits, op); + } -/// Read up to 8 bits from a byte slice starting at a given bit offset. -/// -/// This is similar to `get_8_bits_from_offset` but works with raw byte slices -/// and can read fewer than 8 bits. -/// -/// # Arguments -/// -/// * `slice` - The byte slice to read from -/// * `number_of_bits_to_read` - Number of bits to read (must be ≤ 8) -/// * `bit_offset` - Starting bit offset within the first byte -/// -/// # Returns -/// -/// A u8 containing the requested bits in the least significant positions -#[inline] -fn read_up_to_byte_from_offset( - slice: &[u8], - number_of_bits_to_read: usize, - bit_offset: usize, -) -> u8 { - assert!(number_of_bits_to_read <= 8); - assert_ne!(number_of_bits_to_read, 0); - assert_ne!(slice.len(), 0); - - let number_of_bytes_to_read = bit_util::ceil(number_of_bits_to_read + bit_offset, 8); - - // number of bytes to read - // might be one more than sizeof(u64) if the offset is in the middle of a byte - assert!(slice.len() >= number_of_bytes_to_read, "slice is too small"); - - let mut bits = slice[0] >> bit_offset; - for (i, &byte) in slice - .iter() - .take(number_of_bytes_to_read) - .enumerate() - .skip(1) - { - bits |= byte << (i * 8 - bit_offset); + // Making sure that our guarantee that the length and capacity of the mutable buffer + // will not change is upheld + assert_eq!( + self.len(), + mutable_buffer_len, + "The length of the mutable buffer must not change" + ); + assert_eq!( + self.capacity(), + mutable_buffer_cap, + "The capacity of the mutable buffer must not change" + ); } - - bits & ((1 << number_of_bits_to_read) - 1) } /// Perform bitwise binary operation on byte-aligned buffers (i.e. not offsetting into a middle of a byte). @@ -307,10 +219,10 @@ fn read_up_to_byte_from_offset( /// * `len_in_bits` - Number of bits to process /// * `op` - Binary operation to apply #[inline] -fn mutable_buffer_byte_aligned_bitwise_bin_op_helper( +fn byte_aligned_bitwise_bin_op_helper( left: &mut MutableBuffer, left_offset_in_bits: usize, - right: &impl BufferSupportedRhs, + right: impl AsRef<[u8]>, right_offset_in_bits: usize, len_in_bits: usize, mut op: F, @@ -321,14 +233,14 @@ fn mutable_buffer_byte_aligned_bitwise_bin_op_helper( assert_eq!( left_offset_in_bits % 8, 0, - "left_offset_in_bits must be byte aligned" + "offset_in_bits must be byte aligned" ); // 1. Prepare the buffers let (complete_u64_chunks, remainder_bytes) = U64UnalignedSlice::split(left, left_offset_in_bits, len_in_bits); - let right_chunks = BitChunks::new(right.as_slice(), right_offset_in_bits, len_in_bits); + let right_chunks = BitChunks::new(right.as_ref(), right_offset_in_bits, len_in_bits); assert_eq!( bit_util::ceil(right_chunks.remainder_len(), 8), remainder_bytes.len() @@ -351,6 +263,85 @@ fn mutable_buffer_byte_aligned_bitwise_bin_op_helper( } } +/// Perform bitwise unary operation on byte-aligned buffer. +/// +/// This is the optimized path for byte-aligned unary operations. It processes data in +/// u64 chunks for maximum efficiency, then handles any remainder bits. +/// +/// # Arguments +/// +/// * `buffer` - The mutable buffer (must be byte-aligned) +/// * `offset_in_bits` - Starting bit offset (must be multiple of 8) +/// * `len_in_bits` - Number of bits to process +/// * `op` - Unary operation to apply (e.g., `|a| !a`) +#[inline] +fn byte_aligned_bitwise_unary_op_helper( + buffer: &mut MutableBuffer, + offset_in_bits: usize, + len_in_bits: usize, + mut op: F, +) where + F: FnMut(u64) -> u64, +{ + // Must not reach here if we not byte aligned + assert_eq!(offset_in_bits % 8, 0, "offset_in_bits must be byte aligned"); + + let remainder_len = len_in_bits % 64; + + let (complete_u64_chunks, remainder_bytes) = + U64UnalignedSlice::split(buffer, offset_in_bits, len_in_bits); + + assert_eq!(bit_util::ceil(remainder_len, 8), remainder_bytes.len()); + + // 2. Process complete u64 chunks + complete_u64_chunks.apply_unary_op(&mut op); + + // Handle remainder bits if any + if remainder_len > 0 { + handle_mutable_buffer_remainder_unary(&mut op, remainder_bytes, remainder_len) + } +} + +/// Align to byte boundary by applying operation to bits before the next byte boundary. +/// +/// This function handles non-byte-aligned operations by processing bits from the current +/// position up to the next byte boundary, while preserving all other bits in the byte. +/// +/// # Arguments +/// +/// * `op` - Unary operation to apply +/// * `buffer` - The mutable buffer to modify +/// * `offset_in_bits` - Starting bit offset (not byte-aligned) +fn align_to_byte(buffer: &mut MutableBuffer, op: &mut F, offset_in_bits: usize) +where + F: FnMut(u64) -> u64, +{ + let byte_offset = offset_in_bits / 8; + let bit_offset = offset_in_bits % 8; + + // 1. read the first byte from the buffer + let first_byte: u8 = buffer.as_slice()[byte_offset]; + + // 2. Shift byte by the bit offset, keeping only the relevant bits + let relevant_first_byte = first_byte >> bit_offset; + + // 3. run the op on the first byte only + let result_first_byte = op(relevant_first_byte as u64) as u8; + + // 4. Shift back the result to the original position + let result_first_byte = result_first_byte << bit_offset; + + // 5. Mask the bits that are outside the relevant bits in the byte + // so the bits until bit_offset are 1 and the rest are 0 + let mask_for_first_bit_offset = (1 << bit_offset) - 1; + + let result_first_byte = + (first_byte & mask_for_first_bit_offset) | (result_first_byte & !mask_for_first_bit_offset); + + // 6. write back the result to the buffer + buffer.as_slice_mut()[byte_offset] = result_first_byte; +} + /// Centralized structure to handle a mutable u8 slice as a mutable u64 pointer. /// /// Handle the following: @@ -435,7 +426,7 @@ impl<'a> U64UnalignedSlice<'a> { // SAFETY: We asserted that the iterator length and the current length are the same // and the iterator is not empty, so the pointer is valid unsafe { - self.modify_self(right, &mut map); + self.apply_bin_op(right, &mut map); } // Because this consumes self we don't update the length @@ -449,7 +440,7 @@ impl<'a> U64UnalignedSlice<'a> { // SAFETY: the pointer is valid as we are within the length unsafe { - self.modify_self(right, &mut map); + self.apply_bin_op(right, &mut map); } // Because this consumes self we don't update the length @@ -462,8 +453,9 @@ impl<'a> U64UnalignedSlice<'a> { /// the caller must ensure that the pointer is valid for reads and writes /// #[inline] - unsafe fn modify_self(&mut self, right: u64, mut map: impl FnMut(u64, u64) -> u64) { - // Safety the caller must ensure pointer point to a valid u64 + unsafe fn apply_bin_op(&mut self, right: u64, mut map: impl FnMut(u64, u64) -> u64) { + // SAFETY: The constructor ensures the pointer is valid, + // and as to all modifications in U64UnalignedSlice let current_input = unsafe { self.ptr // Reading unaligned as we came from u8 slice @@ -483,7 +475,7 @@ impl<'a> U64UnalignedSlice<'a> { } /// Modify the underlying u64 data in place using a unary operation. - fn modify(mut self, mut map: impl FnMut(u64) -> u64) { + fn apply_unary_op(mut self, mut map: impl FnMut(u64) -> u64) { if self.len == 0 { return; } @@ -494,7 +486,7 @@ impl<'a> U64UnalignedSlice<'a> { // making sure that the iterator is not empty unsafe { // I hope the function get inlined and the compiler remove the dead right parameter - self.modify_self(0, &mut |left, _| map(left)); + self.apply_bin_op(0, &mut |left, _| map(left)); // Because this consumes self we don't update the length } @@ -508,7 +500,7 @@ impl<'a> U64UnalignedSlice<'a> { // SAFETY: the pointer is valid as we are within the length unsafe { // I hope the function get inlined and the compiler remove the dead right parameter - self.modify_self(0, &mut |left, _| map(left)); + self.apply_bin_op(0, &mut |left, _| map(left)); } // Because this consumes self we don't update the length @@ -524,7 +516,8 @@ impl<'a> U64UnalignedSlice<'a> { /// # Arguments /// /// * `op` - Binary operation to apply -/// * `start_remainder_mut_ptr` - Pointer to the start of remainder bytes +/// * `start_remainder_mut_slice` - slice to the start of remainder bytes +/// the length must be equal to `ceil(remainder_len, 8)` /// * `right_remainder_bits` - Right operand bits /// * `remainder_len` - Number of remainder bits #[inline] @@ -536,13 +529,13 @@ fn handle_mutable_buffer_remainder( ) where F: FnMut(u64, u64) -> u64, { - // Only read from mut pointer the number of remainder bits + // Only read from slice the number of remainder bits let left_remainder_bits = get_remainder_bits(start_remainder_mut_slice, remainder_len); // Apply the operation let rem = op(left_remainder_bits, right_remainder_bits); - // Write only the relevant bits back the result to the mutable pointer + // Write only the relevant bits back the result to the mutable slice set_remainder_bits(start_remainder_mut_slice, rem, remainder_len); } @@ -553,7 +546,8 @@ fn handle_mutable_buffer_remainder( /// /// # Arguments /// -/// * `start_remainder_mut_slice` - the slice of bytes to write the remainder bits to +/// * `start_remainder_mut_slice` - the slice of bytes to write the remainder bits to, +/// the length must be equal to `ceil(remainder_len, 8)` /// * `rem` - The result bits to write /// * `remainder_len` - Number of bits to write #[inline] @@ -564,6 +558,15 @@ fn set_remainder_bits(start_remainder_mut_slice: &mut [u8], rem: u64, remainder_ "start_remainder_mut_slice must not be empty" ); assert!(remainder_len < 64, "remainder_len must be less than 64"); + + // This assertion is to make sure that the last byte in the slice is the boundary byte + // (i.e., the byte that contains both remainder bits and bits outside the remainder) + assert_eq!( + start_remainder_mut_slice.len(), + bit_util::ceil(remainder_len, 8), + "start_remainder_mut_slice length must be equal to ceil(remainder_len, 8)" + ); + // Need to update the remainder bytes in the mutable buffer // but not override the bits outside the remainder @@ -571,6 +574,9 @@ fn set_remainder_bits(start_remainder_mut_slice: &mut [u8], rem: u64, remainder_ // to preserve the bits outside the remainder let rem = { // 1. Read the byte that we will override + // we only read the last byte as we verified that start_remainder_mut_slice length is + // equal to ceil(remainder_len, 8), which means the last byte is the boundary byte + // containing both remainder bits and bits outside the remainder let current = start_remainder_mut_slice .last() // Unwrap as we already validated the slice is not empty @@ -595,7 +601,7 @@ fn set_remainder_bits(start_remainder_mut_slice: &mut [u8], rem: u64, remainder_ current | rem }; - // Write back the result to the mutable pointer + // Write back the result to the mutable slice { let remainder_bytes = bit_util::ceil(remainder_len, 8); @@ -616,13 +622,13 @@ fn set_remainder_bits(start_remainder_mut_slice: &mut [u8], rem: u64, remainder_ } } -/// Read remainder bits from a pointer. +/// Read remainder bits from a slice. /// -/// Reads the specified number of bits from memory and returns them as a u64. +/// Reads the specified number of bits from slice and returns them as a u64. /// /// # Arguments /// -/// * `remainder_ptr` - Pointer to the start of the bits +/// * `remainder` - slice to the start of the bits /// * `remainder_len` - Number of bits to read (must be < 64) /// /// # Returns @@ -647,49 +653,6 @@ fn get_remainder_bits(remainder: &[u8], remainder_len: usize) -> u64 { bits & ((1 << remainder_len) - 1) } -/// Perform bitwise unary operation on byte-aligned buffer. -/// -/// This is the optimized path for byte-aligned unary operations. It processes data in -/// u64 chunks for maximum efficiency, then handles any remainder bits. -/// -/// # Arguments -/// -/// * `left` - The mutable buffer (must be byte-aligned) -/// * `left_offset_in_bits` - Starting bit offset (must be multiple of 8) -/// * `len_in_bits` - Number of bits to process -/// * `op` - Unary operation to apply (e.g., `|a| !a`) -#[inline] -fn mutable_byte_aligned_bitwise_unary_op_helper( - left: &mut MutableBuffer, - left_offset_in_bits: usize, - len_in_bits: usize, - mut op: F, -) where - F: FnMut(u64) -> u64, -{ - // Must not reach here if we not byte aligned - assert_eq!( - left_offset_in_bits % 8, - 0, - "left_offset_in_bits must be byte aligned" - ); - - let remainder_len = len_in_bits % 64; - - let (complete_u64_chunks, remainder_bytes) = - U64UnalignedSlice::split(left, left_offset_in_bits, len_in_bits); - - assert_eq!(bit_util::ceil(remainder_len, 8), remainder_bytes.len()); - - // 2. Process complete u64 chunks - complete_u64_chunks.modify(&mut op); - - // Handle remainder bits if any - if remainder_len > 0 { - handle_mutable_buffer_remainder_unary(&mut op, remainder_bytes, remainder_len) - } -} - /// Handle remainder bits (< 64 bits) for unary operations. /// /// This function processes the bits that don't form a complete u64 chunk, @@ -708,241 +671,21 @@ fn handle_mutable_buffer_remainder_unary( ) where F: FnMut(u64) -> u64, { - // Only read from mut pointer the number of remainder bits + // Only read from the slice the number of remainder bits let left_remainder_bits = get_remainder_bits(start_remainder_mut, remainder_len); // Apply the operation let rem = op(left_remainder_bits); - // Write only the relevant bits back the result to the mutable pointer + // Write only the relevant bits back the result to the slice set_remainder_bits(start_remainder_mut, rem, remainder_len); } -/// Apply a bitwise operation to a mutable buffer and update it in-place. -/// -/// This is the main entry point for unary operations. It handles both byte-aligned -/// and non-byte-aligned cases. -/// -/// The input is treated as a bitmap, meaning that offset and length are specified -/// in number of bits. -/// -/// # Arguments -/// -/// * `buffer` - The mutable buffer to modify -/// * `offset_in_bits` - Starting bit offset -/// * `len_in_bits` - Number of bits to process -/// * `op` - Unary operation to apply (e.g., `|a| !a`) -#[allow( - private_bounds, - reason = "MutableOpsBufferSupportedLhs exposes the inner internals which is the implementor choice and we dont want to leak internals" -)] -pub fn mutable_bitwise_unary_op_helper( - buffer: &mut impl MutableOpsBufferSupportedLhs, - offset_in_bits: usize, - len_in_bits: usize, - mut op: F, -) where - F: FnMut(u64) -> u64, -{ - if len_in_bits == 0 { - return; - } - - let mutable_buffer = buffer.inner_mutable_buffer(); - - let mutable_buffer_len = mutable_buffer.len(); - let mutable_buffer_cap = mutable_buffer.capacity(); - - // offset inside a byte - let left_bit_offset = offset_in_bits % 8; - - let is_mutable_buffer_byte_aligned = left_bit_offset == 0; - - if is_mutable_buffer_byte_aligned { - mutable_byte_aligned_bitwise_unary_op_helper( - mutable_buffer, - offset_in_bits, - len_in_bits, - op, - ); - } else { - align_to_byte(&mut op, mutable_buffer, offset_in_bits); - - // If we are not byte aligned we will read the first few bits - let bits_to_next_byte = 8 - left_bit_offset; - - let offset_in_bits = offset_in_bits + bits_to_next_byte; - let len_in_bits = len_in_bits.saturating_sub(bits_to_next_byte); - - if len_in_bits == 0 { - // Making sure that our guarantee that the length and capacity of the mutable buffer - // will not change is upheld - assert_eq!( - mutable_buffer.len(), - mutable_buffer_len, - "The length of the mutable buffer must not change" - ); - assert_eq!( - mutable_buffer.capacity(), - mutable_buffer_cap, - "The capacity of the mutable buffer must not change" - ); - - return; - } - - // We are now byte aligned - mutable_byte_aligned_bitwise_unary_op_helper( - mutable_buffer, - offset_in_bits, - len_in_bits, - op, - ); - } - - // Making sure that our guarantee that the length and capacity of the mutable buffer - // will not change is upheld - assert_eq!( - mutable_buffer.len(), - mutable_buffer_len, - "The length of the mutable buffer must not change" - ); - assert_eq!( - mutable_buffer.capacity(), - mutable_buffer_cap, - "The capacity of the mutable buffer must not change" - ); -} - -/// Apply a bitwise AND operation to two buffers. -/// -/// The left buffer (mutable) is modified in-place to contain the result. -/// The inputs are treated as bitmaps, meaning that offsets and length are -/// specified in number of bits. -/// -/// # Arguments -/// -/// * `left` - The left mutable buffer (will be modified) -/// * `left_offset_in_bits` - Starting bit offset in the left buffer -/// * `right` - The right buffer -/// * `right_offset_in_bits` - Starting bit offset in the right buffer -/// * `len_in_bits` - Number of bits to process -#[allow( - private_bounds, - reason = "MutableOpsBufferSupportedLhs and BufferSupportedRhs exposes the inner internals which is the implementor choice and we dont want to leak internals" -)] -pub fn mutable_buffer_bin_and( - left: &mut impl MutableOpsBufferSupportedLhs, - left_offset_in_bits: usize, - right: &impl BufferSupportedRhs, - right_offset_in_bits: usize, - len_in_bits: usize, -) { - mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right, - right_offset_in_bits, - len_in_bits, - |a, b| a & b, - ) -} - -/// Apply a bitwise OR operation to two buffers. -/// -/// The left buffer (mutable) is modified in-place to contain the result. -/// The inputs are treated as bitmaps, meaning that offsets and length are -/// specified in number of bits. -/// -/// # Arguments -/// -/// * `left` - The left mutable buffer (will be modified) -/// * `left_offset_in_bits` - Starting bit offset in the left buffer -/// * `right` - The right buffer -/// * `right_offset_in_bits` - Starting bit offset in the right buffer -/// * `len_in_bits` - Number of bits to process -#[allow( - private_bounds, - reason = "MutableOpsBufferSupportedLhs and BufferSupportedRhs exposes the inner internals which is the implementor choice and we dont want to leak internals" -)] -pub fn mutable_buffer_bin_or( - left: &mut impl MutableOpsBufferSupportedLhs, - left_offset_in_bits: usize, - right: &impl BufferSupportedRhs, - right_offset_in_bits: usize, - len_in_bits: usize, -) { - mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right, - right_offset_in_bits, - len_in_bits, - |a, b| a | b, - ) -} - -/// Apply a bitwise XOR operation to two buffers. -/// -/// The left buffer (mutable) is modified in-place to contain the result. -/// The inputs are treated as bitmaps, meaning that offsets and length are -/// specified in number of bits. -/// -/// # Arguments -/// -/// * `left` - The left mutable buffer (will be modified) -/// * `left_offset_in_bits` - Starting bit offset in the left buffer -/// * `right` - The right buffer -/// * `right_offset_in_bits` - Starting bit offset in the right buffer -/// * `len_in_bits` - Number of bits to process -#[allow( - private_bounds, - reason = "MutableOpsBufferSupportedLhs and BufferSupportedRhs exposes the inner internals which is the implementor choice and we dont want to leak internals" -)] -pub fn mutable_buffer_bin_xor( - left: &mut impl MutableOpsBufferSupportedLhs, - left_offset_in_bits: usize, - right: &impl BufferSupportedRhs, - right_offset_in_bits: usize, - len_in_bits: usize, -) { - mutable_bitwise_bin_op_helper( - left, - left_offset_in_bits, - right, - right_offset_in_bits, - len_in_bits, - |a, b| a ^ b, - ) -} - -/// Apply a bitwise NOT operation to the passed buffer. -/// -/// The buffer (mutable) is modified in-place to contain the result. -/// The input is treated as bitmap, meaning that offsets and length are -/// specified in number of bits. -/// -/// # Arguments -/// -/// * `buffer` - The mutable buffer (will be modified) -/// * `offset_in_bits` - Starting bit offset in the buffer -/// * `len_in_bits` - Number of bits to process -#[allow( - private_bounds, - reason = "MutableOpsBufferSupportedLhs exposes the inner internals which is the implementor choice and we dont want to leak internals" -)] -pub fn mutable_buffer_unary_not( - buffer: &mut impl MutableOpsBufferSupportedLhs, - offset_in_bits: usize, - len_in_bits: usize, -) { - mutable_bitwise_unary_op_helper(buffer, offset_in_bits, len_in_bits, |a| !a) -} - #[cfg(test)] mod tests { use crate::bit_iterator::BitIterator; use crate::{BooleanBuffer, BooleanBufferBuilder}; + use rand::Rng; fn test_mutable_buffer_bin_op_helper( left_data: &[bool], @@ -967,8 +710,9 @@ mod tests { .map(|(l, r)| expected_op(*l, *r)) .collect(); - super::mutable_bitwise_bin_op_helper( - &mut left_buffer, + let mutable_buffer = unsafe { left_buffer.mutable_buffer() }; + + mutable_buffer.bitwise_binary_op( left_offset_in_bits, right_buffer.inner(), right_offset_in_bits, @@ -1005,7 +749,9 @@ mod tests { .map(|b| expected_op(*b)) .collect(); - super::mutable_bitwise_unary_op_helper(&mut buffer, offset_in_bits, len_in_bits, op); + let mutable_buffer = unsafe { buffer.mutable_buffer() }; + + mutable_buffer.bitwise_unary_op(offset_in_bits, len_in_bits, op); let result: Vec = BitIterator::new(buffer.as_slice(), offset_in_bits, len_in_bits).collect(); @@ -1019,8 +765,9 @@ mod tests { // Helper to create test data of specific length fn create_test_data(len: usize) -> (Vec, Vec) { - let left: Vec = (0..len).map(|i| i % 2 == 0).collect(); - let right: Vec = (0..len).map(|i| (i / 2) % 2 == 0).collect(); + let mut rng = rand::rng(); + let left: Vec = (0..len).map(|_| rng.random_bool(0.5)).collect(); + let right: Vec = (0..len).map(|_| rng.random_bool(0.5)).collect(); (left, right) } @@ -1151,6 +898,12 @@ mod tests { test_all_binary_ops(&left, &right, 3, 7, 50); } + #[test] + fn test_binary_ops_offsets_greater_than_8_less_than_64() { + let (left, right) = create_test_data(200); + test_all_binary_ops(&left, &right, 13, 27, 100); + } + // ===== NOT (Unary) Operation Tests ===== #[test] diff --git a/arrow-buffer/src/builder/boolean.rs b/arrow-buffer/src/builder/boolean.rs index ff8a1f66a303..37738c638343 100644 --- a/arrow-buffer/src/builder/boolean.rs +++ b/arrow-buffer/src/builder/boolean.rs @@ -15,11 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::{ - BooleanBuffer, Buffer, MutableBuffer, MutableOpsBufferSupportedLhs, bit_mask, bit_util, - mutable_buffer_bin_and, mutable_buffer_bin_or, mutable_buffer_bin_xor, - mutable_buffer_unary_not, -}; +use crate::{BooleanBuffer, Buffer, MutableBuffer, bit_mask, bit_util}; use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not, Range}; /// Builder for [`BooleanBuffer`] @@ -248,6 +244,16 @@ impl BooleanBufferBuilder { self.buffer.as_slice_mut() } + /// Return a mutable reference to the internal buffer + /// + /// # Safety + /// The caller must ensure that any modifications to the buffer maintain the invariant + /// `self.len < buffer.len() / 8` (that is that the buffer has enough capacity to hold `self.len` bits). + #[inline] + pub unsafe fn mutable_buffer(&mut self) -> &mut MutableBuffer { + &mut self.buffer + } + /// Creates a [`BooleanBuffer`] #[inline] pub fn finish(&mut self) -> BooleanBuffer { @@ -262,18 +268,11 @@ impl BooleanBufferBuilder { } } -/// This trait is not public API so it does not leak the inner mutable buffer -impl MutableOpsBufferSupportedLhs for BooleanBufferBuilder { - fn inner_mutable_buffer(&mut self) -> &mut MutableBuffer { - &mut self.buffer - } -} - impl Not for BooleanBufferBuilder { type Output = BooleanBufferBuilder; fn not(mut self) -> Self::Output { - mutable_buffer_unary_not(&mut self.buffer, 0, self.len); + self.buffer.bitwise_unary_op(0, self.len, |a| !a); Self { buffer: self.buffer, len: self.len, @@ -305,7 +304,8 @@ impl BitAndAssign<&BooleanBuffer> for BooleanBufferBuilder { fn bitand_assign(&mut self, rhs: &BooleanBuffer) { assert_eq!(self.len, rhs.len()); - mutable_buffer_bin_and(&mut self.buffer, 0, rhs.inner(), rhs.offset(), self.len); + self.buffer + .bitwise_binary_op(0, rhs.inner(), rhs.offset(), self.len, |a, b| a & b); } } @@ -313,7 +313,8 @@ impl BitAndAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { fn bitand_assign(&mut self, rhs: &BooleanBufferBuilder) { assert_eq!(self.len, rhs.len()); - mutable_buffer_bin_and(&mut self.buffer, 0, &rhs.buffer, 0, self.len); + self.buffer + .bitwise_binary_op(0, rhs.as_slice(), 0, self.len, |a, b| a & b); } } @@ -341,7 +342,8 @@ impl BitOrAssign<&BooleanBuffer> for BooleanBufferBuilder { fn bitor_assign(&mut self, rhs: &BooleanBuffer) { assert_eq!(self.len, rhs.len()); - mutable_buffer_bin_or(&mut self.buffer, 0, rhs.inner(), rhs.offset(), self.len); + self.buffer + .bitwise_binary_op(0, rhs.inner(), rhs.offset(), self.len, |a, b| a | b); } } @@ -349,7 +351,8 @@ impl BitOrAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { fn bitor_assign(&mut self, rhs: &BooleanBufferBuilder) { assert_eq!(self.len, rhs.len()); - mutable_buffer_bin_or(&mut self.buffer, 0, &rhs.buffer, 0, self.len); + self.buffer + .bitwise_binary_op(0, rhs.as_slice(), 0, self.len, |a, b| a | b); } } @@ -377,7 +380,8 @@ impl BitXorAssign<&BooleanBuffer> for BooleanBufferBuilder { fn bitxor_assign(&mut self, rhs: &BooleanBuffer) { assert_eq!(self.len, rhs.len()); - mutable_buffer_bin_xor(&mut self.buffer, 0, rhs.inner(), rhs.offset(), self.len); + self.buffer + .bitwise_binary_op(0, rhs.inner(), rhs.offset(), self.len, |a, b| a ^ b); } } @@ -385,7 +389,8 @@ impl BitXorAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { fn bitxor_assign(&mut self, rhs: &BooleanBufferBuilder) { assert_eq!(self.len, rhs.len()); - mutable_buffer_bin_xor(&mut self.buffer, 0, &rhs.buffer, 0, self.len); + self.buffer + .bitwise_binary_op(0, rhs.as_slice(), 0, self.len, |a, b| a ^ b); } } diff --git a/arrow-buffer/src/util/bit_util.rs b/arrow-buffer/src/util/bit_util.rs index f00a33aca96d..638c8c7ef26f 100644 --- a/arrow-buffer/src/util/bit_util.rs +++ b/arrow-buffer/src/util/bit_util.rs @@ -94,6 +94,55 @@ pub fn ceil(value: usize, divisor: usize) -> usize { value.div_ceil(divisor) } +/// Read up to 8 bits from a byte slice starting at a given bit offset. +/// +/// # Arguments +/// +/// * `slice` - The byte slice to read from +/// * `number_of_bits_to_read` - Number of bits to read (must be < 8) +/// * `bit_offset` - Starting bit offset within the first byte (must be < 8) +/// +/// # Returns +/// +/// A `u8` containing the requested bits in the least significant positions +/// +/// # Panics +/// - Panics if `number_of_bits_to_read` is 0 or >= 8 +/// - Panics if `bit_offset` is >= 8 +/// - Panics if `slice` is empty or too small to read the requested bits +/// +#[inline] +pub(crate) fn read_up_to_byte_from_offset( + slice: &[u8], + number_of_bits_to_read: usize, + bit_offset: usize, +) -> u8 { + assert!(number_of_bits_to_read < 8, "can read up to 8 bits only"); + assert!(bit_offset < 8, "bit offset must be less than 8"); + assert_ne!( + number_of_bits_to_read, 0, + "number of bits to read must be greater than 0" + ); + assert_ne!(slice.len(), 0, "slice must not be empty"); + + let number_of_bytes_to_read = ceil(number_of_bits_to_read + bit_offset, 8); + + // number of bytes to read + assert!(slice.len() >= number_of_bytes_to_read, "slice is too small"); + + let mut bits = slice[0] >> bit_offset; + for (i, &byte) in slice + .iter() + .take(number_of_bytes_to_read) + .enumerate() + .skip(1) + { + bits |= byte << (i * 8 - bit_offset); + } + + bits & ((1 << number_of_bits_to_read) - 1) +} + #[cfg(test)] mod tests { use std::collections::HashSet; @@ -279,4 +328,77 @@ mod tests { assert_eq!(ceil(10, 10000000000), 1); assert_eq!(ceil(10000000000, 1000000000), 10); } + + #[test] + fn test_read_up_to() { + let all_ones = &[0b10111001, 0b10001100]; + + for (bit_offset, expected) in [ + (0, 0b00000001), + (1, 0b00000000), + (2, 0b00000000), + (3, 0b00000001), + (4, 0b00000001), + (5, 0b00000001), + (6, 0b00000000), + (7, 0b00000001), + ] { + let result = read_up_to_byte_from_offset(all_ones, 1, bit_offset); + assert_eq!( + result, expected, + "failed at bit_offset {bit_offset}. result, expected:\n{result:08b}\n{expected:08b}" + ); + } + + for (bit_offset, expected) in [ + (0, 0b00000001), + (1, 0b00000000), + (2, 0b00000010), + (3, 0b00000011), + (4, 0b00000011), + (5, 0b00000001), + (6, 0b00000010), + (7, 0b00000001), + ] { + let result = read_up_to_byte_from_offset(all_ones, 2, bit_offset); + assert_eq!( + result, expected, + "failed at bit_offset {bit_offset}. result, expected:\n{result:08b}\n{expected:08b}" + ); + } + + for (bit_offset, expected) in [ + (0, 0b00111001), + (1, 0b00011100), + (2, 0b00101110), + (3, 0b00010111), + (4, 0b00001011), + (5, 0b00100101), + (6, 0b00110010), + (7, 0b00011001), + ] { + let result = read_up_to_byte_from_offset(all_ones, 6, bit_offset); + assert_eq!( + result, expected, + "failed at bit_offset {bit_offset}. result, expected:\n{result:08b}\n{expected:08b}" + ); + } + + for (bit_offset, expected) in [ + (0, 0b00111001), + (1, 0b01011100), + (2, 0b00101110), + (3, 0b00010111), + (4, 0b01001011), + (5, 0b01100101), + (6, 0b00110010), + (7, 0b00011001), + ] { + let result = read_up_to_byte_from_offset(all_ones, 7, bit_offset); + assert_eq!( + result, expected, + "failed at bit_offset {bit_offset}. result, expected:\n{result:08b}\n{expected:08b}" + ); + } + } } From 0a64bcb8d90bf81ef6566dab3a9aa4ec4f143916 Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Sun, 26 Oct 2025 22:50:40 +0200 Subject: [PATCH 15/15] add comments --- arrow-buffer/src/builder/boolean.rs | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/arrow-buffer/src/builder/boolean.rs b/arrow-buffer/src/builder/boolean.rs index 37738c638343..577368f76d13 100644 --- a/arrow-buffer/src/builder/boolean.rs +++ b/arrow-buffer/src/builder/boolean.rs @@ -283,6 +283,10 @@ impl Not for BooleanBufferBuilder { impl BitAnd<&BooleanBuffer> for BooleanBufferBuilder { type Output = BooleanBufferBuilder; + /// Performs a bitwise AND operation between this buffer and `rhs`, returning the result as a new buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitand(mut self, rhs: &BooleanBuffer) -> Self::Output { self &= rhs; @@ -293,6 +297,10 @@ impl BitAnd<&BooleanBuffer> for BooleanBufferBuilder { impl BitAnd<&BooleanBufferBuilder> for BooleanBufferBuilder { type Output = BooleanBufferBuilder; + /// Performs a bitwise AND operation between this buffer and `rhs`, returning the result as a new buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitand(mut self, rhs: &BooleanBufferBuilder) -> Self::Output { self &= rhs; @@ -301,6 +309,10 @@ impl BitAnd<&BooleanBufferBuilder> for BooleanBufferBuilder { } impl BitAndAssign<&BooleanBuffer> for BooleanBufferBuilder { + /// Performs a bitwise AND operation between this buffer and `rhs`, storing the result in this buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitand_assign(&mut self, rhs: &BooleanBuffer) { assert_eq!(self.len, rhs.len()); @@ -310,6 +322,10 @@ impl BitAndAssign<&BooleanBuffer> for BooleanBufferBuilder { } impl BitAndAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { + /// Performs a bitwise AND operation between this buffer and `rhs`, storing the result in this buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitand_assign(&mut self, rhs: &BooleanBufferBuilder) { assert_eq!(self.len, rhs.len()); @@ -321,6 +337,10 @@ impl BitAndAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { impl BitOr<&BooleanBuffer> for BooleanBufferBuilder { type Output = BooleanBufferBuilder; + /// Performs a bitwise OR operation between this buffer and `rhs`, returning the result as a new buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitor(mut self, rhs: &BooleanBuffer) -> Self::Output { self |= rhs; @@ -331,6 +351,10 @@ impl BitOr<&BooleanBuffer> for BooleanBufferBuilder { impl BitOr<&BooleanBufferBuilder> for BooleanBufferBuilder { type Output = BooleanBufferBuilder; + /// Performs a bitwise OR operation between this buffer and `rhs`, returning the result as a new buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitor(mut self, rhs: &BooleanBufferBuilder) -> Self::Output { self |= rhs; @@ -339,6 +363,10 @@ impl BitOr<&BooleanBufferBuilder> for BooleanBufferBuilder { } impl BitOrAssign<&BooleanBuffer> for BooleanBufferBuilder { + /// Performs a bitwise OR operation between this buffer and `rhs`, storing the result in this buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitor_assign(&mut self, rhs: &BooleanBuffer) { assert_eq!(self.len, rhs.len()); @@ -348,6 +376,10 @@ impl BitOrAssign<&BooleanBuffer> for BooleanBufferBuilder { } impl BitOrAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { + /// Performs a bitwise OR operation between this buffer and `rhs`, storing the result in this buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitor_assign(&mut self, rhs: &BooleanBufferBuilder) { assert_eq!(self.len, rhs.len()); @@ -359,6 +391,10 @@ impl BitOrAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { impl BitXor<&BooleanBuffer> for BooleanBufferBuilder { type Output = BooleanBufferBuilder; + /// Performs a bitwise XOR operation between this buffer and `rhs`, returning the result as a new buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitxor(mut self, rhs: &BooleanBuffer) -> Self::Output { self ^= rhs; @@ -369,6 +405,10 @@ impl BitXor<&BooleanBuffer> for BooleanBufferBuilder { impl BitXor<&BooleanBufferBuilder> for BooleanBufferBuilder { type Output = BooleanBufferBuilder; + /// Performs a bitwise XOR operation between this buffer and `rhs`, returning the result as a new buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitxor(mut self, rhs: &BooleanBufferBuilder) -> Self::Output { self ^= rhs; @@ -377,6 +417,10 @@ impl BitXor<&BooleanBufferBuilder> for BooleanBufferBuilder { } impl BitXorAssign<&BooleanBuffer> for BooleanBufferBuilder { + /// Performs a bitwise XOR operation between this buffer and `rhs`, storing the result in this buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitxor_assign(&mut self, rhs: &BooleanBuffer) { assert_eq!(self.len, rhs.len()); @@ -386,6 +430,10 @@ impl BitXorAssign<&BooleanBuffer> for BooleanBufferBuilder { } impl BitXorAssign<&BooleanBufferBuilder> for BooleanBufferBuilder { + /// Performs a bitwise XOR operation between this buffer and `rhs`, storing the result in this buffer. + /// + /// # Panics + /// Panics if the lengths of the two buffers are not equal fn bitxor_assign(&mut self, rhs: &BooleanBufferBuilder) { assert_eq!(self.len, rhs.len());