Skip to content
1 change: 1 addition & 0 deletions _quarto.yml
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,7 @@ website:
contents:
- docs/overview.qmd
- docs/user-interfaces.qmd
- docs/reading-data.qmd
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm thinking the articles that come out of this can go in their own section (in between Reactivity and Syntax Modes)

- section: "<span class='emoji-icon'>🤖</span> __Generative AI__"
contents:
- docs/genai-inspiration.qmd
Expand Down
260 changes: 260 additions & 0 deletions docs/persistent-storage.qmd
Original file line number Diff line number Diff line change
@@ -0,0 +1,260 @@
---
title: Persistent storage & writing data
editor:
markdown:
wrap: sentence
lightbox:
effect: fade
---

## What is persistent storage and when do we need it?

User data collected and saved somewhere across sessions
Ex. Users submit a form

Not:
Data store we read from for display only (see Reading Data article)


Note: We're not talking about saving user login information or authentication.

CALLOUT: If you are a Connect user, Connect has support for both Auth and supports secrets for storage. Link to those docs.

## Working with non-database stores (ex. Google Sheets, Object Stores)

```python
from shiny.express import ui, render, input
from shiny import reactive
import polars as pl
import gspread

gc = gspread.service_account(filename="../.config/gspread/service_account.json")
url = "https://docs.google.com/spreadsheets/d/166uMLi75GLD7eFUKe-vdKIpCk0EwbYWa2md4Zn9vJO4/edit?gid=0#gid=0"

sheet = gc.open_by_url(url)
worksheet = sheet.get_worksheet(0)
last_updated = None

with ui.sidebar():
ui.input_text("name_input", "Enter your name:", placeholder="Your name here")
ui.input_checkbox("checkbox", "I like checkboxes")
ui.input_slider("slider", "My favorite number is:", min=0, max=100, value=50)
ui.input_action_button("submit_button", "Submit")
print(worksheet.get_all_records())

flag = reactive.value(0)
df = reactive.value(
pl.from_dicts(
worksheet.get_all_records(expected_headers=["name", "checkbox", "number"]),
schema={"name": pl.Utf8, "checkbox": pl.String, "number": pl.Int64},
)
)


@reactive.effect
@reactive.event(input.submit_button)
def add_row():
new_row = [input.name_input(), input.checkbox(), input.slider()]
worksheet.append_row(new_row, insert_data_option="INSERT_ROWS")
flag.set(1)


@reactive.effect
def refresh_data_if_dirty():
if flag() == 1:
df.set(
pl.from_dicts(
worksheet.get_all_records(),
schema={"name": pl.Utf8, "checkbox": pl.String, "number": pl.Int64},
)
)
print("Data is dirty, refreshing...")
flag.set(0)


@render.data_frame
def show_results():
return render.DataGrid(df())

```


CALLOUT: pins does this other stuff, plus has some reactive support

You might think of potential concerns with lots of parallel users or nonblocking setting
use this to motivate DB section


## Working with Databases

### Reading from and writing to databases

We'll ground this initial example in an app that reads, writes, and updats the visualization based on a simple flag that tells us if the data is dirty or not.


::: {.panel-tabset .panel-pills}

#### Polars & Postgres
```{.python filename="setup.py"}
import polars as pl

# Don't store your creds in plain text, load them from a environment variables or a secrets manager
uri = "postgresql://postgres@localhost:5432/template1"

# Notice that this uses `read_*`. Polars cannot read the data lazily from the database, so this will load the entire query into memory.
def run_query(query):
return pl.read_database_uri(query, uri)

```

```{.python filename="app.py"}
from shiny.express import ui, render, input, app_opts
from shiny import reactive
import polars as pl
import load_data

app_opts(debug=True)

full_query = "SELECT * FROM testapp"

with ui.sidebar():
ui.input_text("name_input", "Enter your name:", placeholder="Your name here")
ui.input_checkbox("checkbox", "I like checkboxes")
ui.input_slider("slider", "My favorite number is:", min=0, max=100, value=50)
ui.input_action_button("submit_button", "Submit")

# If this value is 1, our data is dirty, if it is 0, our data is clean
flag = reactive.value(0)
df = reactive.value(pl.LazyFrame(load_data.run_query(full_query)))


@reactive.effect
@reactive.event(input.submit_button)
def add_row():
new_row = pl.DataFrame(
{
"name": input.name_input(),
"checkbox": input.checkbox(),
"favorite_number": input.slider(),
}
)
new_row.write_database("testapp", load_data.uri, if_table_exists="append")

# The table we're displaying no longer reflects the reality of the database because we just added a row, so we mark the data as dirty.

flag.set(1)
print("Added a new row!")


@reactive.effect
def refresh_data_if_dirty():
if flag() == 1:
df.set(pl.LazyFrame(load_data.run_query(full_query)))
print("Data is dirty, refreshing...")
flag.set(0)


@render.data_frame
def show_results():
return render.DataGrid(df().collect())

```
:::


Introduce the motivation for polling using the above example

### Reactive Polling

### App examples with polling
::: {.panel-tabset .panel-pills}

#### Polars + Postgres
```{.python filename="setup.py"}
import polars as pl

# Don't store your creds in plain text, load them from a environment variables or a secrets manager
uri = "postgresql://postgres@localhost:5432/template1"


# Notice that this uses `read_*`. Polars cannot read the data lazily from the database, so this will load the entire query into memory.
def run_query(query):
return pl.read_database_uri(query, uri)


```

```{.python filename="app.py"}

from shiny.express import ui, render, input, app_opts
from shiny import reactive
import polars as pl
import setup

app_opts(debug=True)

full_query = "SELECT * FROM testapp"
starting_data = pl.LazyFrame(setup.run_query(full_query))

with ui.sidebar():
ui.input_text("name_input", "Enter your name:", placeholder="Your name here")
ui.input_checkbox("checkbox", "I like checkboxes")
ui.input_slider("slider", "My favorite number is:", min=0, max=100, value=50)
ui.input_action_button("submit_button", "Submit")


@reactive.effect
@reactive.event(input.submit_button)
def add_row():
new_row = pl.DataFrame(
{
"name": input.name_input(),
"checkbox": input.checkbox(),
"favorite_number": input.slider(),
}
)
new_row.write_database("testapp", setup.uri, if_table_exists="append")
print("Added a new row!")


def num_rows():
return load_data.run_query("SELECT COUNT(*) FROM testapp").item()


@reactive.poll(num_rows, 1)
def get_data():
print("Checking for new data...")
data = setup.update_data()
return data


@render.data_frame
def show_results():
return render.DataGrid(get_data())

```
:::





## Connecting to databases

### Example connecting with Ibis and saving to db with a form

## Deployment options

### Auth - connect callout

### SQL injection prevention
Callout near wherever we first execute SQL, be careful, diff libraries have tooling


### Connection pooling & transaction locking?

### Note on credentials? (don't use your admin creds in your app, have user creds for db, scope them narrowly, etc.)




Loading
Loading