Skip to content

Commit

Permalink
Merge branch 'main' into emilk/arrow-deserializer
Browse files Browse the repository at this point in the history
  • Loading branch information
emilk committed Jan 6, 2025
2 parents aceb048 + cef0ffe commit 77e4543
Show file tree
Hide file tree
Showing 25 changed files with 412 additions and 355 deletions.
9 changes: 7 additions & 2 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -5668,6 +5668,7 @@ version = "0.22.0-alpha.1+dev"
dependencies = [
"ahash",
"anyhow",
"arrow",
"criterion",
"document-features",
"indent",
Expand Down Expand Up @@ -5722,7 +5723,6 @@ dependencies = [
"egui_plot",
"itertools 0.13.0",
"nohash-hasher",
"re_byte_size",
"re_data_source",
"re_data_ui",
"re_format",
Expand Down Expand Up @@ -6524,20 +6524,25 @@ dependencies = [
name = "re_ui"
version = "0.22.0-alpha.1+dev"
dependencies = [
"arrow",
"eframe",
"egui",
"egui_commonmark",
"egui_extras",
"egui_kittest",
"egui_tiles",
"itertools 0.13.0",
"once_cell",
"parking_lot",
"rand",
"re_arrow2",
"re_byte_size",
"re_entity_db",
"re_format",
"re_log",
"re_log_types",
"re_tracing",
"re_types",
"serde",
"serde_json",
"smallvec",
Expand Down Expand Up @@ -6628,6 +6633,7 @@ dependencies = [
"itertools 0.13.0",
"re_chunk_store",
"re_dataframe",
"re_error",
"re_format",
"re_log",
"re_log_types",
Expand Down Expand Up @@ -6906,7 +6912,6 @@ dependencies = [
"directories",
"egui",
"egui-wgpu",
"egui_extras",
"egui_tiles",
"emath",
"glam",
Expand Down
6 changes: 4 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -383,8 +383,10 @@ rust_2018_idioms = { level = "warn", priority = -1 }
rust_2021_prelude_collisions = "warn"
semicolon_in_expressions_from_macros = "warn"
trivial_numeric_casts = "warn"
unsafe_op_in_unsafe_fn = "warn" # `unsafe_op_in_unsafe_fn` may become the default in future Rust versions: https://github.com/rust-lang/rust/issues/71668
unexpected_cfgs = "deny"
unsafe_op_in_unsafe_fn = "warn" # `unsafe_op_in_unsafe_fn` may become the default in future Rust versions: https://github.com/rust-lang/rust/issues/71668
unexpected_cfgs = { level = "deny", check-cfg = [
'cfg(TODO)', # NOLINT
] } # allow `#[cfg(TODO)]` to compile (it will still fail CI)
unused_extern_crates = "warn"
unused_import_braces = "warn"
unused_lifetimes = "warn"
Expand Down
1 change: 1 addition & 0 deletions crates/store/re_chunk_store/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ re_types_core.workspace = true
# External dependencies:
ahash.workspace = true
anyhow.workspace = true
arrow.workspace = true
arrow2 = { workspace = true, features = ["compute_concatenate"] }
document-features.workspace = true
indent.workspace = true
Expand Down
1 change: 1 addition & 0 deletions crates/store/re_chunk_store/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ pub use re_chunk::{
pub use re_log_types::{ResolvedTimeRange, TimeInt, TimeType, Timeline};

pub mod external {
pub use arrow;
pub use arrow2;

pub use re_chunk;
Expand Down
43 changes: 38 additions & 5 deletions crates/store/re_dataframe/src/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ use itertools::Itertools;

use nohash_hasher::{IntMap, IntSet};
use re_chunk::{
Chunk, ComponentName, EntityPath, RangeQuery, RowId, TimeInt, Timeline, UnitChunkShared,
external::arrow::array::ArrayRef, Chunk, ComponentName, EntityPath, RangeQuery, RowId, TimeInt,
Timeline, UnitChunkShared,
};
use re_chunk_store::{
ChunkStore, ColumnDescriptor, ColumnSelector, ComponentColumnDescriptor,
Expand Down Expand Up @@ -794,7 +795,39 @@ impl<E: StorageEngineLike> QueryHandle<E> {
/// }
/// ```
#[inline]
pub fn next_row(&self) -> Option<Vec<Box<dyn Arrow2Array>>> {
pub fn next_row(&self) -> Option<Vec<ArrayRef>> {
self.engine
.with(|store, cache| self._next_row(store, cache))
.map(|vec| vec.into_iter().map(|a| a.into()).collect())
}

/// Returns the next row's worth of data.
///
/// The returned vector of Arrow arrays strictly follows the schema specified by [`Self::schema`].
/// Columns that do not yield any data will still be present in the results, filled with null values.
///
/// Each cell in the result corresponds to the latest _locally_ known value at that particular point in
/// the index, for each respective `ColumnDescriptor`.
/// See [`QueryExpression::sparse_fill_strategy`] to go beyond local resolution.
///
/// Example:
/// ```ignore
/// while let Some(row) = query_handle.next_row() {
/// // …
/// }
/// ```
///
/// ## Pagination
///
/// Use [`Self::seek_to_row`]:
/// ```ignore
/// query_handle.seek_to_row(42);
/// for row in query_handle.into_iter().take(len) {
/// // …
/// }
/// ```
#[inline]
fn next_row_arrow2(&self) -> Option<Vec<Box<dyn Arrow2Array>>> {
self.engine
.with(|store, cache| self._next_row(store, cache))
}
Expand Down Expand Up @@ -1239,7 +1272,7 @@ impl<E: StorageEngineLike> QueryHandle<E> {
pub fn next_row_batch(&self) -> Option<RecordBatch> {
Some(RecordBatch {
schema: self.schema().clone(),
data: Arrow2Chunk::new(self.next_row()?),
data: Arrow2Chunk::new(self.next_row_arrow2()?),
})
}

Expand All @@ -1266,13 +1299,13 @@ impl<E: StorageEngineLike> QueryHandle<E> {
/// Returns an iterator backed by [`Self::next_row`].
#[allow(clippy::should_implement_trait)] // we need an anonymous closure, this won't work
pub fn iter(&self) -> impl Iterator<Item = Vec<Box<dyn Arrow2Array>>> + '_ {
std::iter::from_fn(move || self.next_row())
std::iter::from_fn(move || self.next_row_arrow2())
}

/// Returns an iterator backed by [`Self::next_row`].
#[allow(clippy::should_implement_trait)] // we need an anonymous closure, this won't work
pub fn into_iter(self) -> impl Iterator<Item = Vec<Box<dyn Arrow2Array>>> {
std::iter::from_fn(move || self.next_row())
std::iter::from_fn(move || self.next_row_arrow2())
}

/// Returns an iterator backed by [`Self::next_row_batch`].
Expand Down
49 changes: 45 additions & 4 deletions crates/top/re_sdk/src/recording_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -977,10 +977,10 @@ impl RecordingStream {
/// [SDK Micro Batching]: https://www.rerun.io/docs/reference/sdk/micro-batching
/// [component bundle]: [`AsComponents`]
#[inline]
pub fn log(
pub fn log<AS: ?Sized + AsComponents>(
&self,
ent_path: impl Into<EntityPath>,
as_components: &impl AsComponents,
as_components: &AS,
) -> RecordingStreamResult<()> {
self.log_with_static(ent_path, false, as_components)
}
Expand Down Expand Up @@ -1108,11 +1108,11 @@ impl RecordingStream {
/// [SDK Micro Batching]: https://www.rerun.io/docs/reference/sdk/micro-batching
/// [component bundle]: [`AsComponents`]
#[inline]
pub fn log_with_static(
pub fn log_with_static<AS: ?Sized + AsComponents>(
&self,
ent_path: impl Into<EntityPath>,
static_: bool,
as_components: &impl AsComponents,
as_components: &AS,
) -> RecordingStreamResult<()> {
let row_id = RowId::new(); // Create row-id as early as possible. It has a timestamp and is used to estimate e2e latency.
self.log_component_batches_impl(
Expand Down Expand Up @@ -2286,6 +2286,7 @@ impl RecordingStream {
#[cfg(test)]
mod tests {
use re_chunk::TransportChunk;
use re_log_types::example_components::MyLabel;

use super::*;

Expand Down Expand Up @@ -2623,4 +2624,44 @@ mod tests {

vec![row0, row1, row2]
}

// See <https://github.com/rerun-io/rerun/pull/8587> for context.
#[test]
fn allows_ascomponents_unsized() {
let labels = [
MyLabel("a".into()),
MyLabel("b".into()),
MyLabel("c".into()),
];

let (rec, _mem) = RecordingStreamBuilder::new("rerun_example_test_ascomponents_unsized")
.default_enabled(false)
.enabled(false)
.memory()
.unwrap();

// This call used to *not* compile due to a lack of `?Sized` bounds.
rec.log("labels", &labels as &dyn crate::AsComponents)
.unwrap();
}

// See <https://github.com/rerun-io/rerun/pull/8587> for context.
#[test]
fn allows_componentbatch_unsized() {
let labels = [
MyLabel("a".into()),
MyLabel("b".into()),
MyLabel("c".into()),
];

let (rec, _mem) = RecordingStreamBuilder::new("rerun_example_test_componentbatch_unsized")
.default_enabled(false)
.enabled(false)
.memory()
.unwrap();

// This call used to *not* compile due to a lack of `?Sized` bounds.
rec.log("labels", &labels as &dyn crate::ComponentBatch)
.unwrap();
}
}
2 changes: 1 addition & 1 deletion crates/viewer/re_chunk_store_ui/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ re_chunk_store.workspace = true
re_format.workspace = true
re_log_types.workspace = true
re_types.workspace = true
re_ui.workspace = true
re_ui = { workspace = true, features = ["arrow"] }
re_viewer_context.workspace = true

egui.workspace = true
Expand Down
82 changes: 0 additions & 82 deletions crates/viewer/re_chunk_store_ui/src/arrow_ui.rs

This file was deleted.

2 changes: 1 addition & 1 deletion crates/viewer/re_chunk_store_ui/src/chunk_ui.rs
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ impl ChunkUi {
chunk.component_batch_raw(&component_desc.component_name, row_index);
match component_data {
Some(Ok(data)) => {
crate::arrow_ui::arrow2_ui(ui, &*data);
re_ui::arrow2_ui(ui, re_ui::UiLayout::List, &*data);
}
Some(Err(err)) => {
ui.error_with_details_on_hover(err.to_string());
Expand Down
1 change: 0 additions & 1 deletion crates/viewer/re_chunk_store_ui/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
//! Browser UI for [`re_chunk_store::ChunkStore`].
mod arrow_ui;
mod chunk_list_mode;
mod chunk_store_ui;
mod chunk_ui;
Expand Down
3 changes: 1 addition & 2 deletions crates/viewer/re_component_ui/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ workspace = true
all-features = true

[dependencies]
re_byte_size.workspace = true
re_data_source.workspace = true
re_data_ui.workspace = true # Needed for `item_ui`.
re_format.workspace = true
Expand All @@ -30,7 +29,7 @@ re_types = { workspace = true, features = [
"egui_plot", # Needed to draw marker shapes.
] }
re_types_core.workspace = true
re_ui.workspace = true
re_ui = { workspace = true, features = ["arrow"] }
re_viewer_context.workspace = true

arrow.workspace = true
Expand Down
Loading

0 comments on commit 77e4543

Please sign in to comment.