diff --git a/_quarto.yml b/_quarto.yml
index cfb4a24b..97bde757 100644
--- a/_quarto.yml
+++ b/_quarto.yml
@@ -251,6 +251,7 @@ website:
contents:
- docs/overview.qmd
- docs/user-interfaces.qmd
+ - docs/reading-data.qmd
- section: "🤖 __Generative AI__"
contents:
- docs/genai-inspiration.qmd
diff --git a/docs/persistent-storage.qmd b/docs/persistent-storage.qmd
new file mode 100644
index 00000000..2aef13eb
--- /dev/null
+++ b/docs/persistent-storage.qmd
@@ -0,0 +1,260 @@
+---
+title: Persistent storage & writing data
+editor:
+ markdown:
+ wrap: sentence
+lightbox:
+ effect: fade
+---
+
+## What is persistent storage and when do we need it?
+
+User data collected and saved somewhere across sessions
+Ex. Users submit a form
+
+Not:
+Data store we read from for display only (see Reading Data article)
+
+
+Note: We're not talking about saving user login information or authentication.
+
+CALLOUT: If you are a Connect user, Connect has support for both Auth and supports secrets for storage. Link to those docs.
+
+## Working with non-database stores (ex. Google Sheets, Object Stores)
+
+```python
+from shiny.express import ui, render, input
+from shiny import reactive
+import polars as pl
+import gspread
+
+gc = gspread.service_account(filename="../.config/gspread/service_account.json")
+url = "https://docs.google.com/spreadsheets/d/166uMLi75GLD7eFUKe-vdKIpCk0EwbYWa2md4Zn9vJO4/edit?gid=0#gid=0"
+
+sheet = gc.open_by_url(url)
+worksheet = sheet.get_worksheet(0)
+last_updated = None
+
+with ui.sidebar():
+ ui.input_text("name_input", "Enter your name:", placeholder="Your name here")
+ ui.input_checkbox("checkbox", "I like checkboxes")
+ ui.input_slider("slider", "My favorite number is:", min=0, max=100, value=50)
+ ui.input_action_button("submit_button", "Submit")
+print(worksheet.get_all_records())
+
+flag = reactive.value(0)
+df = reactive.value(
+ pl.from_dicts(
+ worksheet.get_all_records(expected_headers=["name", "checkbox", "number"]),
+ schema={"name": pl.Utf8, "checkbox": pl.String, "number": pl.Int64},
+ )
+)
+
+
+@reactive.effect
+@reactive.event(input.submit_button)
+def add_row():
+ new_row = [input.name_input(), input.checkbox(), input.slider()]
+ worksheet.append_row(new_row, insert_data_option="INSERT_ROWS")
+ flag.set(1)
+
+
+@reactive.effect
+def refresh_data_if_dirty():
+ if flag() == 1:
+ df.set(
+ pl.from_dicts(
+ worksheet.get_all_records(),
+ schema={"name": pl.Utf8, "checkbox": pl.String, "number": pl.Int64},
+ )
+ )
+ print("Data is dirty, refreshing...")
+ flag.set(0)
+
+
+@render.data_frame
+def show_results():
+ return render.DataGrid(df())
+
+```
+
+
+CALLOUT: pins does this other stuff, plus has some reactive support
+
+You might think of potential concerns with lots of parallel users or nonblocking setting
+use this to motivate DB section
+
+
+## Working with Databases
+
+### Reading from and writing to databases
+
+We'll ground this initial example in an app that reads, writes, and updats the visualization based on a simple flag that tells us if the data is dirty or not.
+
+
+::: {.panel-tabset .panel-pills}
+
+#### Polars & Postgres
+```{.python filename="setup.py"}
+import polars as pl
+
+# Don't store your creds in plain text, load them from a environment variables or a secrets manager
+uri = "postgresql://postgres@localhost:5432/template1"
+
+# Notice that this uses `read_*`. Polars cannot read the data lazily from the database, so this will load the entire query into memory.
+def run_query(query):
+ return pl.read_database_uri(query, uri)
+
+```
+
+```{.python filename="app.py"}
+from shiny.express import ui, render, input, app_opts
+from shiny import reactive
+import polars as pl
+import load_data
+
+app_opts(debug=True)
+
+full_query = "SELECT * FROM testapp"
+
+with ui.sidebar():
+ ui.input_text("name_input", "Enter your name:", placeholder="Your name here")
+ ui.input_checkbox("checkbox", "I like checkboxes")
+ ui.input_slider("slider", "My favorite number is:", min=0, max=100, value=50)
+ ui.input_action_button("submit_button", "Submit")
+
+# If this value is 1, our data is dirty, if it is 0, our data is clean
+flag = reactive.value(0)
+df = reactive.value(pl.LazyFrame(load_data.run_query(full_query)))
+
+
+@reactive.effect
+@reactive.event(input.submit_button)
+def add_row():
+ new_row = pl.DataFrame(
+ {
+ "name": input.name_input(),
+ "checkbox": input.checkbox(),
+ "favorite_number": input.slider(),
+ }
+ )
+ new_row.write_database("testapp", load_data.uri, if_table_exists="append")
+
+ # The table we're displaying no longer reflects the reality of the database because we just added a row, so we mark the data as dirty.
+
+ flag.set(1)
+ print("Added a new row!")
+
+
+@reactive.effect
+def refresh_data_if_dirty():
+ if flag() == 1:
+ df.set(pl.LazyFrame(load_data.run_query(full_query)))
+ print("Data is dirty, refreshing...")
+ flag.set(0)
+
+
+@render.data_frame
+def show_results():
+ return render.DataGrid(df().collect())
+
+```
+:::
+
+
+Introduce the motivation for polling using the above example
+
+### Reactive Polling
+
+### App examples with polling
+::: {.panel-tabset .panel-pills}
+
+#### Polars + Postgres
+```{.python filename="setup.py"}
+import polars as pl
+
+# Don't store your creds in plain text, load them from a environment variables or a secrets manager
+uri = "postgresql://postgres@localhost:5432/template1"
+
+
+# Notice that this uses `read_*`. Polars cannot read the data lazily from the database, so this will load the entire query into memory.
+def run_query(query):
+ return pl.read_database_uri(query, uri)
+
+
+```
+
+```{.python filename="app.py"}
+
+from shiny.express import ui, render, input, app_opts
+from shiny import reactive
+import polars as pl
+import setup
+
+app_opts(debug=True)
+
+full_query = "SELECT * FROM testapp"
+starting_data = pl.LazyFrame(setup.run_query(full_query))
+
+with ui.sidebar():
+ ui.input_text("name_input", "Enter your name:", placeholder="Your name here")
+ ui.input_checkbox("checkbox", "I like checkboxes")
+ ui.input_slider("slider", "My favorite number is:", min=0, max=100, value=50)
+ ui.input_action_button("submit_button", "Submit")
+
+
+@reactive.effect
+@reactive.event(input.submit_button)
+def add_row():
+ new_row = pl.DataFrame(
+ {
+ "name": input.name_input(),
+ "checkbox": input.checkbox(),
+ "favorite_number": input.slider(),
+ }
+ )
+ new_row.write_database("testapp", setup.uri, if_table_exists="append")
+ print("Added a new row!")
+
+
+def num_rows():
+ return load_data.run_query("SELECT COUNT(*) FROM testapp").item()
+
+
+@reactive.poll(num_rows, 1)
+def get_data():
+ print("Checking for new data...")
+ data = setup.update_data()
+ return data
+
+
+@render.data_frame
+def show_results():
+ return render.DataGrid(get_data())
+
+```
+:::
+
+
+
+
+
+## Connecting to databases
+
+### Example connecting with Ibis and saving to db with a form
+
+## Deployment options
+
+### Auth - connect callout
+
+### SQL injection prevention
+Callout near wherever we first execute SQL, be careful, diff libraries have tooling
+
+
+### Connection pooling & transaction locking?
+
+### Note on credentials? (don't use your admin creds in your app, have user creds for db, scope them narrowly, etc.)
+
+
+
+
diff --git a/docs/reading-data.qmd b/docs/reading-data.qmd
new file mode 100644
index 00000000..5a2bf669
--- /dev/null
+++ b/docs/reading-data.qmd
@@ -0,0 +1,371 @@
+---
+title: Reading data
+editor:
+ markdown:
+ wrap: sentence
+lightbox:
+ effect: fade
+---
+
+From local files, databases, cloud-hosted data stores, and beyond, if you can read your data into Python, you can also read it into Shiny. Here we'll highlight some of our recommended ways to read data, which can be grouped into two main categories: [eager](#eager) and [lazy](#lazy). By eager, we mean loading all your data into memory when the app first starts. By lazy, we mean loading _portions_ of data into memory _as needed_. The eager approach is generally recommended for small to moderately sized datasets -- for larger data, lazy loading may be necessary, but it will add complexity to your app logic.
+
+## Eager loading {#eager}
+
+Loading all your data into memory when your app first starts is the simplest way to work with data in Shiny.
+This makes it easy create UI from data, and more generally reason about your app's data logic. However, this also means that before a user can interact with your app, the data must be loaded into memory, so be mindful to keep this step as fast and efficient as possible (or consider [lazy](#lazy) loading).
+
+### From a file
+
+If your data lives in a file (e.g., CSV, Parquet, etc.), you can read it into memory using a variety of tools. Popular libraries for this include [Polars](https://pola.rs/), [DuckDB](https://duckdb.org/), and [Pandas](https://pandas.pydata.org/):
+
+::: {.panel-tabset .panel-pills}
+
+#### Polars
+
+[Polars](https://pola.rs/) is a fantastic library that makes data manipulation fast and intuitive. We recommend starting here for most data analysis tasks in Shiny apps.
+
+```python
+import polars as pl
+dat = pl.read_csv(Path(__file__).parent / "my_data.csv")
+```
+
+#### DuckDB
+
+[DuckDB](https://duckdb.org/) is a fast analytical database system. It's a great choice to quickly import data into memory.
+
+```python
+import duckdb
+dat = duckdb.read_csv(Path(__file__).parent / "my_data.csv")
+```
+:::
+
+#### Pandas
+
+[Pandas](https://pandas.pydata.org/) is a widely used data manipulation library in Python. While it may not be as fast and ergonomic as Polars for large datasets, it is still a solid choice for many apps.
+
+```python
+import pandas as pd
+dat = pd.read_csv(Path(__file__).parent / "my_data.csv")
+```
+
+
+### From a database
+
+If your data lives in a database, reading it into memory can be done using similar tools like Polars. Most databases have a connection interface via [SQLAlchemy](https://www.sqlalchemy.org/), which has integration with packages like Polars (and Pandas):
+
+```python
+import polars as pl
+dat = pl.read_database_uri(
+ "postgresql://user:password@hostname/database_name",
+ "SELECT * FROM tablename"
+)
+```
+
+See [SQLAlchemy's documentation](https://docs.sqlalchemy.org/en/20/core/engines.html) for more information on connection strings for different databases.
+
+
+::: callout-tip
+### Multiple tables
+
+`pl.read_database_uri()` opens a connection to the database, runs the provided query, and then closes the connection. If you need to read multiple tables, consider using a database-centric library like [Ibis](https://ibis-project.org/) to manage the connection and queries more effectively.
+We'll cover this (explicitly opening/closing connections) more in the [lazy loading](#lazy) section below.
+:::
+
+
+### From a cloud store
+
+
+
+### Performance tips
+
+When eagerly reading data into memory, it's important to consider the performance implications. Here are some tips to help keep your app fast and responsive:
+
+#### Express
+
+When in Express mode (i.e., using `shiny.express`), we highly recommended reading data (or other expensive setup code) in a separate module, then import it into your `app.py` file. This ensures the data is loaded only once, improving performance. For more details, refer to the Express documentation [here](express-in-depth.qmd#shared-objects).
+
+```{.python filename="setup.py"}
+import polars as pl
+from pathlib import Path
+
+dat = pl.read_csv(Path(__file__).parent / "my_data.csv")
+```
+
+```{.python filename="app.py"}
+from shiny.express import render
+from setup import dat
+
+@render.data_frame
+def df():
+ return dat
+```
+
+#### Large data
+
+If you have larger data, eagerly reading all of it into memory may not be feasible, so you may need a smarter approach than loading it all into memory when your app first starts. The next section covers some more advanced techniques for lazily loading data into memory as needed.
+
+However, before reaching for lazy loading (which will add complexity to your app), consider the following optimizations for reading data into memory:
+
+1. **File format**: Use efficient file formats like Parquet instead of CSV for large datasets.
+2. **ETL process**: Preprocess and clean your data before loading it into your app to reduce size and complexity.
+3. **Database optimization**: If reading from a database, ensure it's optimized for analytical queries (e.g., using indexes, partitioning, etc.).
+
+## Lazy reading {#lazy}
+
+Some datasets are too costly to read into memory when an app first loads. In this situation, we can leverage tools to lazily load data into memory as needed, helping to keep our app fast and responsive.
+
+### From a file
+
+A fantastic tool for lazily reading data from a file is [Polars' Lazy API](https://docs.pola.rs/user-guide/lazy/). This way, you express data manipulations, but don't actually perform them until you need to `.collect()` results. This looks something roughly like this:
+
+```python
+import polars as pl
+from shiny.express import render
+from pathlib import Path
+
+# Executes instantly (it doesn't load the data into memory)
+dat = pl.scan_parquet(Path(__file__).parent / "test_data.parquet")
+
+@render.data_frame
+def df():
+ # Express manipulations and .collect() only when needed
+ return dat.head(100).collect()
+```
+
+If you have a choice, we recommend using Polars' Lazy API over running queries against a database, as it is often simpler and faster. That said, there are certainly cases where a database is more appropriate or necessary.
+
+
+### From a database
+
+Some fantastic tools for lazy loading data from a database are [Ibis](https://ibis-project.org/) and [SQLAlchemy](https://www.sqlalchemy.org/). With these tools, you can connnect to practically any database, and express data manipulations in Python (or SQL with SQLAlchemy), which are only executed when you call `.execute()`. When using these tools, it's important to explicitly open and close the database connection to avoid connection leaks. Here's an example using Ibis with a PostgreSQL database:
+
+```bash
+pip install 'ibis-framework[postgres]'
+```
+
+```python
+import ibis
+from shiny.express import render, session
+
+# Connect to the database (nearly instant)
+con = ibis.postgres.connect(
+ user="username",
+ password="password",
+ host="hostname",
+ port=5432,
+ database="database",
+)
+dat = con.table("tablename")
+
+# Cleanup connection when the session ends
+_ = session.on_ended(con.close)
+
+@render.data_frame
+def df():
+ # Perform manipulations and .execute() only when needed
+ return dat.head(100).execute()
+```
+
+::: callout-tip
+### Using SQL directly
+
+If you prefer to write SQL more directly, use SQLAlchemy to connect to your database and execute SQL queries. This approach provides more flexibility but requires more manual handling of SQL queries.
+
+
+Show example code
+
+```python
+from shiny.express import render
+from sqlalchemy import create_engine, text
+
+engine = create_engine('postgresql://user:password@hostname/database_name')
+with engine.connect() as conn:
+ dat = conn.execute(text("SELECT * FROM tablename LIMIT 100"))
+```
+
+:::
+
+### From a cloud store
+
+
+
+
+### A real example
+
+Let's look at an example with the [The Weather Dataset](https://www.kaggle.com/datasets/guillemservera/global-daily-climate-data). This data is fairly large (nearly 28M rows), but by loading it lazily, we can keep our app responsive by only loading one city+season data at a time.
+
+::: {.panel-tabset .panel-pills}
+
+#### Polars (file)
+
+```python
+import polars as pl
+
+from shiny.express import ui, render, input
+
+# Use `scan_*` instead of `read_*` to use the lazy API
+dat = pl.scan_parquet("./daily_weather.parquet")
+
+with ui.sidebar():
+ ui.input_checkbox_group(
+ "season",
+ "Season",
+ choices=["Summer", "Winter", "Fall", "Spring"],
+ selected="Summer",
+ )
+ # Import just the unique city names for our selectize input
+ cities = dat.select("city_name").unique().collect().to_series().to_list()
+ ui.input_selectize("city", "City", choices=cities)
+
+# Store manipulation in a reactive calc
+# (convenient for writing once and using in multiple places)
+@reactive.calc
+def filtered_dat():
+ return (
+ dat
+ .filter(pl.col("city_name") == input.city())
+ .filter(pl.col("season").is_in(input.season()))
+ )
+
+# Display the filtered data
+@render.data_frame
+def results_df():
+ return filtered_dat().collect()
+```
+
+#### Ibis (database)
+
+```python
+import ibis
+from shiny.express import ui, render, input, reactive, session
+
+# Connect to the database (quick, doesn't load data)
+con = ibis.postgres.connect(
+ user="", password="", host="", port=, database=""
+)
+dat = con.table("weather")
+_ = session.on_ended(con.close)
+
+with ui.sidebar():
+ ui.input_checkbox_group(
+ "season",
+ "Season",
+ choices=["Summer", "Winter", "Autumn", "Spring"],
+ selected="Summer",
+ )
+ # Import just the unique city names for our selectize input
+ cities = dat.select("city_name").distinct().execute().tolist()
+ ui.input_selectize("city", "City", choices=cities)
+
+
+# Store data manipulations in a reactive calculation
+# (convenient when using the data in multiple places)
+@reactive.calc
+def filtered_dat():
+ return dat.filter(
+ [_.city_name == input.city(), _.season.isin(input.season())]
+ )
+
+# Display the filtered data
+@render.data_frame
+def results_df():
+ return filtered_dat().execute()
+```
+:::
+
+
+## Examples
+
+We have numerous full examples and starter templates of dashboards using different datastores. We've sorted them into categories roughly corresponding to the tech stack at the bottom of this article.
+
+::: {.panel-tabset .panel-pills}
+#### DuckDB
+
+::: {layout-ncol=2}
+##### Query Explorer Template
+
+
+[See the code](https://shiny.posit.co/py/templates/database-explorer/)
+
+##### Identify Outliers App
+
+
+[See the code](https://github.com/skaltman/outliers-app-db-python/)
+:::
+
+#### Pandas
+
+::: {layout-ncol=2}
+##### Restaurant Tips Template
+
+
+[See the code](https://shiny.posit.co/py/templates/dashboard-tips/)
+
+##### AWS Community Builders App
+
+
+[See the code](https://github.com/robertgv/aws-community-builders-dashboard)
+:::
+
+:::
+
+## Reactive reading {#reactive}
+
+In some cases, you may need your app to occasionally re-read an updated data source _while the app is running_. This is a more advanced use case, but Shiny provides tools to help with this scenario.
+
+### From a file
+
+If your data lives in a local file, use [`reactive.file_reader()`](https://shiny.posit.co/py/api/express/reactive.file_reader.html) to monitor the file for changes and re-read it when it changes.
+By default, `reactive.file_reader()` checks the file's modification time every second, but you can adjust this with the `interval` argument.
+
+```python
+import polars as pl
+from shiny import reactive
+from shiny.express import render
+
+file = pathlib.Path(__file__).parent / "mtcars.csv"
+@reactive.file_reader(file)
+def read_file():
+ return pl.read_csv(file)
+
+@render.data_frame
+def result():
+ return read_file()
+```
+
+### From a database
+
+If your data lives in a database, use [`reactive.poll()`](https://shiny.posit.co/py/api/express/reactive.poll.html) to periodically check if the data has changed and re-read it when it has.
+With `reactive.poll()`, you're required to provide a function to check if the data has changed, which should be as efficient as possible.
+Unfortunately, there is no universal way to check if data has changed in a database, so you'll need to implement this logic based on your specific use case.
+In general, you might check a "last updated" timestamp column:
+
+```python
+import ibis
+from shiny.express import reactive
+
+con = ibis.postgres.connect(user="", password="", host="", port=, database="")
+table = con.table("tablename")
+
+def check_last_updated():
+ return table.last_updated.max().execute()
+
+# Every 5 seconds, check if the max timestamp has changed
+@reactive.poll(check_last_updated, interval=5)
+def data():
+ return table.execute()
+```
+
+::: callout-note
+### Polling to export to file
+
+Since Polars is so good at (lazily) reading data from files, it's tempting to export from a database to a file. In this case, you could set up a `reactive.poll()` to save the database table as a file whenever it changes, and `reactive.file_reader()` to re-read the file when the file changes.
+:::
+
+
+For a deeper dive into reactively reading "streaming" data, see the [Streaming Data](../templates/monitor-database/index.qmd) template.
+
+
+## What about saving data?
+
+In some cases, you may need to save data from your application back to a database or other remote location. This is a more advanced use case, but Shiny provides tools to help with this scenario. In the next article, we'll cover some strategies for persisting data from your Shiny app.