Skip to content

Commit cd97d3b

Browse files
andyorangeabeschornerVFisleofdeath-afk
authored
iterate on nested dataframe and remove some to_pandas calls (#39)
* new demo with pandas like interface * added md file --------- Co-authored-by: abeschornerVF <andreas.beschorner@vodafone.com> Co-authored-by: Andreas Beschorner <isleofdeath@gmail.com>
1 parent 283766b commit cd97d3b

5 files changed

Lines changed: 1051 additions & 27 deletions

File tree

‎demos/demo_compare_pd_ibis.py‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Compare Leanframe workflows: pandas-first vs strict-Ibis-until-output.
4+
5+
This demo answers two practical questions in one place:
6+
7+
1) How to work with Leanframe when starting from pandas vs staying in Ibis.
8+
2) A runnable side-by-side example using a tiny nested dataset.
9+
10+
Scope:
11+
- Simple data model (two columns, one nested) for each table.
12+
- Same operations in both approaches:
13+
- data creation
14+
- nesting + extraction (NestedHandler.prepare)
15+
- join on extracted nested fields
16+
- indexing (.set_index + .head)
17+
- materialize to pandas only for final display
18+
"""
19+
20+
import sys
21+
import uuid
22+
from pathlib import Path
23+
24+
sys.path.insert(0, str(Path(__file__).parent.parent))
25+
26+
import ibis
27+
import pandas as pd
28+
import pyarrow as pa
29+
30+
import leanframe
31+
from leanframe.core.frame import DataFrame
32+
from leanframe.core.nested_handler import NestedHandler
33+
34+
35+
def print_concept_comparison() -> None:
36+
"""Explain practical tradeoffs between pandas-first and Ibis-first workflows."""
37+
print("\n" + "=" * 80)
38+
print("Leanframe Workflow Comparison: pandas-first vs strict-Ibis")
39+
print("=" * 80)
40+
41+
print("\n1) Data Creation")
42+
print("- pandas-first:")
43+
print(" Build pandas DataFrames first, then call session.DataFrame(pandas_df).")
44+
print(" Best when your source is already local/in-memory or notebook-centric.")
45+
print("- strict-Ibis:")
46+
print(" Build Arrow/Ibis tables and read them via session.read_ibis(...).")
47+
print(" Best for backend-native execution (BigQuery-style workflow).")
48+
49+
print("\n2) Nested Columns")
50+
print("- Both paths use NestedHandler identically once data is in Leanframe DataFrames.")
51+
print("- Use handler.prepare(name) to flatten nested fields into underscore columns.")
52+
print(" Example: profile.email -> profile_email")
53+
54+
print("\n3) Joins")
55+
print("- pandas-first:")
56+
print(" Easy startup, but you materialize data locally early.")
57+
print("- strict-Ibis:")
58+
print(" Keep execution in backend expressions as long as possible.")
59+
print(" Better alignment with warehouse engines and larger datasets.")
60+
61+
print("\n4) Indexing")
62+
print("- Indexing API is the same in both approaches: set_index(), iloc, loc, head/tail.")
63+
print("- Indexing composes well after nested extraction and joins.")
64+
65+
print("\n5) When to use to_pandas()")
66+
print("- pandas-first:")
67+
print(" You already started in pandas, so conversion happened at ingestion.")
68+
print("- strict-Ibis:")
69+
print(" Use to_pandas() only at the very end (display/export/debug sample).")
70+
71+
print("\n6) What Leanframe already feels like")
72+
print("- DataFrame: columns, dtypes, assign, set_index, head, tail, loc, iloc")
73+
print("- Series: sum, mean, min, max, count, isin, dtype, to_list")
74+
print("- NestedHandler: join() and prepare() provide the SQL/Ibis bridge")
75+
print("- Current gap vs pandas: no full merge/groupby/query/drop/reset_index parity yet")
76+
print("- Practical takeaway: write pandas-shaped code, but keep execution deferred in Ibis")
77+
78+
79+
def build_pandas_inputs() -> tuple[pd.DataFrame, pd.DataFrame]:
80+
"""Create tiny nested inputs in pandas for the pandas-first path."""
81+
customers_pd = pd.DataFrame(
82+
{
83+
"customer_id": [1, 2, 3],
84+
"profile": [
85+
{"email": "alice@example.com", "age": 30},
86+
{"email": "bob@example.com", "age": 25},
87+
{"email": "charlie@example.com", "age": 35},
88+
],
89+
}
90+
)
91+
92+
orders_pd = pd.DataFrame(
93+
{
94+
"order_id": [101, 102, 103],
95+
"shipping": [
96+
{"recipient": {"email": "alice@example.com"}},
97+
{"recipient": {"email": "bob@example.com"}},
98+
{"recipient": {"email": "alice@example.com"}},
99+
],
100+
}
101+
)
102+
103+
return customers_pd, orders_pd
104+
105+
106+
def build_ibis_inputs(backend: ibis.BaseBackend) -> tuple[DataFrame, DataFrame]:
107+
"""Create the same logical data in Arrow/Ibis for the strict-Ibis path."""
108+
customers_tbl = backend.create_table(
109+
f"customers_cmp_{uuid.uuid4().hex[:8]}",
110+
pa.Table.from_pydict(
111+
{
112+
"customer_id": [1, 2, 3],
113+
"profile": [
114+
{"email": "alice@example.com", "age": 30},
115+
{"email": "bob@example.com", "age": 25},
116+
{"email": "charlie@example.com", "age": 35},
117+
],
118+
},
119+
schema=pa.schema(
120+
[
121+
pa.field("customer_id", pa.int64()),
122+
pa.field(
123+
"profile",
124+
pa.struct(
125+
[pa.field("email", pa.string()), pa.field("age", pa.int64())]
126+
),
127+
),
128+
]
129+
),
130+
),
131+
temp=True,
132+
)
133+
134+
orders_tbl = backend.create_table(
135+
f"orders_cmp_{uuid.uuid4().hex[:8]}",
136+
pa.Table.from_pydict(
137+
{
138+
"order_id": [101, 102, 103],
139+
"shipping": [
140+
{"recipient": {"email": "alice@example.com"}},
141+
{"recipient": {"email": "bob@example.com"}},
142+
{"recipient": {"email": "alice@example.com"}},
143+
],
144+
},
145+
schema=pa.schema(
146+
[
147+
pa.field("order_id", pa.int64()),
148+
pa.field(
149+
"shipping",
150+
pa.struct(
151+
[
152+
pa.field(
153+
"recipient",
154+
pa.struct([pa.field("email", pa.string())]),
155+
)
156+
]
157+
),
158+
),
159+
]
160+
),
161+
),
162+
temp=True,
163+
)
164+
165+
return DataFrame(customers_tbl), DataFrame(orders_tbl)
166+
167+
168+
def run_pipeline(label: str, customers_df: DataFrame, orders_df: DataFrame) -> None:
169+
"""Run the same nesting, join, and indexing pipeline for either input style."""
170+
print("\n" + "-" * 80)
171+
print(f"Pipeline: {label}")
172+
print("-" * 80)
173+
174+
handler = NestedHandler()
175+
handler.add("customers", customers_df)
176+
handler.add("orders", orders_df)
177+
178+
# NestedHandler already gives a pandas-like join wrapper over Ibis.
179+
joined_df = handler.join(
180+
tables={"c": "customers", "o": "orders"},
181+
on=[("c", "profile_email", "o", "shipping_recipient_email")],
182+
how="inner",
183+
)
184+
185+
print(f"Joined rows: {joined_df['order_id'].count()}")
186+
print(f"Joined columns: {joined_df.columns.tolist()}")
187+
188+
# Keep the workflow pandas-shaped: index first, then take head().
189+
by_age_desc = joined_df.set_index("profile_age", ascending=False)
190+
top_rows = by_age_desc.head(5)
191+
192+
print(f"Oldest matched customer age: {joined_df['profile_age'].max()}")
193+
194+
# Final materialization for output.
195+
result_pd = top_rows.to_pandas()[
196+
["customer_id", "profile_email", "profile_age", "order_id"]
197+
]
198+
print("\nFinal output (pandas, materialized at the end):")
199+
print(result_pd)
200+
201+
202+
def main() -> None:
203+
print_concept_comparison()
204+
205+
backend = ibis.duckdb.connect()
206+
session = leanframe.Session(backend=backend)
207+
208+
try:
209+
# Approach A: pandas-first ingestion.
210+
customers_pd, orders_pd = build_pandas_inputs()
211+
customers_from_pd = session.DataFrame(customers_pd)
212+
orders_from_pd = session.DataFrame(orders_pd)
213+
run_pipeline("A) pandas-first", customers_from_pd, orders_from_pd)
214+
215+
# Approach B: strict-Ibis ingestion (Arrow/Ibis until final output).
216+
customers_ibis, orders_ibis = build_ibis_inputs(backend)
217+
customers_from_ibis = session.read_ibis(customers_ibis.to_ibis())
218+
orders_from_ibis = session.read_ibis(orders_ibis.to_ibis())
219+
run_pipeline(
220+
"B) strict-Ibis (pandas only at the end)",
221+
customers_from_ibis,
222+
orders_from_ibis,
223+
)
224+
finally:
225+
backend.disconnect()
226+
227+
228+
if __name__ == "__main__":
229+
main()

‎demos/demo_flexible_joins.py‎

Lines changed: 48 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,12 @@
3939

4040
import sys
4141
from pathlib import Path
42+
import uuid
4243

4344
sys.path.insert(0, str(Path(__file__).parent.parent))
4445

4546
import ibis
46-
import pandas as pd
47+
import pyarrow as pa
4748
import leanframe
4849
from leanframe.core.frame import DataFrame
4950
from leanframe.core.nested_handler import NestedHandler
@@ -74,20 +75,36 @@ def main():
7475
customers_df = create_customers_for_join()
7576
orders_df = create_orders_for_join()
7677

78+
# Keep data in Ibis/Arrow space and register everything on the demo backend.
79+
customers_tbl = backend.create_table(
80+
f"customers_demo_{uuid.uuid4().hex[:8]}",
81+
customers_df.to_ibis().to_pyarrow(),
82+
temp=True,
83+
)
84+
orders_tbl = backend.create_table(
85+
f"orders_demo_{uuid.uuid4().hex[:8]}",
86+
orders_df.to_ibis().to_pyarrow(),
87+
temp=True,
88+
)
89+
7790
# Add third table - products
78-
products_pd = pd.DataFrame(
79-
{
80-
"product_id": [1, 2, 3],
81-
"name": ["Widget", "Gadget", "Doohickey"],
82-
"category": ["Electronics", "Electronics", "Hardware"],
83-
"price": [29.99, 149.99, 9.99],
84-
}
91+
products_tbl = backend.create_table(
92+
f"products_demo_{uuid.uuid4().hex[:8]}",
93+
pa.Table.from_pydict(
94+
{
95+
"product_id": [1, 2, 3],
96+
"name": ["Widget", "Gadget", "Doohickey"],
97+
"category": ["Electronics", "Electronics", "Hardware"],
98+
"price": [29.99, 149.99, 9.99],
99+
}
100+
),
101+
temp=True,
85102
)
86-
products_df = session.DataFrame(products_pd)
103+
products_df = session.read_ibis(products_tbl)
87104

88105
# Add to NestedHandler
89-
nested.add("customers", session.DataFrame(customers_df.to_pandas()))
90-
nested.add("orders", session.DataFrame(orders_df.to_pandas()))
106+
nested.add("customers", session.read_ibis(customers_tbl))
107+
nested.add("orders", session.read_ibis(orders_tbl))
91108
nested.add("products", products_df)
92109

93110
print("\n✅ Added 3 tables:")
@@ -154,13 +171,27 @@ def main():
154171
print("=" * 70)
155172

156173
print("\n🎯 Goal: customers ⋈ orders ⋈ products")
157-
print(" Old methods: ❌ Can't do this!")
158-
print(" New approach: ✅ Easy!")
159174

160-
# Modify orders to have product_id
161-
orders_with_products_pd = orders_flat.to_pandas()
162-
orders_with_products_pd["product_id"] = [1, 2, 1, 3, 2] # Match order IDs
163-
orders_with_products = session.DataFrame(orders_with_products_pd)
175+
# Modify orders to have product_id without materializing to pandas.
176+
order_product_map_tbl = backend.create_table(
177+
f"order_product_map_demo_{uuid.uuid4().hex[:8]}",
178+
pa.Table.from_pydict(
179+
{
180+
"map_order_id": [101, 102, 103, 104, 105],
181+
"product_id": [1, 2, 1, 3, 2],
182+
}
183+
),
184+
temp=True,
185+
)
186+
orders_with_products = DataFrame(
187+
orders_flat._data.join(
188+
order_product_map_tbl,
189+
predicates=[
190+
orders_flat._data.order_id == order_product_map_tbl.map_order_id
191+
],
192+
how="inner",
193+
).drop(order_product_map_tbl.map_order_id)
194+
)
164195

165196
print("\n1️⃣ First join: customers ⋈ orders")
166197
step1 = customers_flat._data.join(

0 commit comments

Comments
 (0)