Data Docs (#320)

* Adding pages

* Updates so far

* Updates

* Updates

* additional updates

* Current draft

* Current

* Updates after chat

* Updates

* Notes plus remove old modules

* More updates

* Updates made

* Updated with apps

* Overhaul reading data

* Updates to persistent storage

* Updated content

* additional context

* Progress

* both ibis examples added

* More updates

* Updates

* Connect info

* Added link

* Switching order

* Correction

* Added notif

* Simplified

* wip updates to persistent data article

* Added reading from remote

* finish brain dump on persistent data

* Remove link in Essentials section

* Small edits

* Corrections to first example

* Corrected GoogleSheets example

* Smoothed out the string/boolean thing

* Restoring paste error in setup for sheets

* Removed try except at start

* Updates to dotenv

* Update docs/reading-data.qmd

* Minor updates to wording

* small changes/improvements

* QA on code up to cloud store

* More corrections to reading data

* More correcdtions to reaction section

* Final corrections for ibis

* small changes/improvements

* Update docs/persistent-storage.qmd

* Updating s3

* Moving some examples to separate app folder, add screenshot

* Changing examples

* Minor grammar edits to reading data

* Minor grammar edits for persistent storage

* Updating template pages

---------

Co-authored-by: Carson <[email protected]>

Author: elnelson575

_quarto.yml

@@ -271,6 +271,10 @@ website:
             - docs/reactive-foundations.qmd
             - docs/reactive-patterns.qmd
             - docs/reactive-mutable.qmd
+        - section: "<span class='emoji-icon'>🗃️</span>  __Data__"
+          contents:
+            - docs/reading-data.qmd
+            - docs/persistent-storage.qmd
         - section: "<span class='emoji-icon'>📝</span>  __Syntax modes__"
           contents:
             - docs/express-vs-core.qmd

docs/apps/persistent-storage/app.py

@@ -0,0 +1,37 @@
+import polars as pl
+from setup import append_info, load_data, save_info
+from shiny import reactive
+from shiny.express import app_opts, input, render, ui
+
+with ui.sidebar():
+    ui.input_text("name_input", "Enter your name", placeholder="Your name here")
+    ui.input_checkbox("checkbox", "I like checkboxes")
+    ui.input_slider("slider", "My favorite number is:", min=0, max=100, value=50)
+    ui.input_action_button("submit_button", "Submit")
+
+# Load the initial data into a reactive value when the app starts
+data = reactive.value(load_data())
+
+
+# Append new user data on submit
+@reactive.effect
+@reactive.event(input.submit_button)
+def submit_data():
+    info = {
+        "name": input.name_input(),
+        "checkbox": input.checkbox(),
+        "favorite_number": input.slider(),
+    }
+    # Update the (in-memory) data
+    d = data()
+    data.set(append_info(d, info))
+    # Save info to persistent storage (out-of-memory)
+    save_info(info)
+    # Provide some user feedback
+    ui.notification_show("Submitted, thanks!")
+
+
+# Data grid that shows the current data
+@render.data_frame
+def show_results():
+    return render.DataGrid(data())

docs/apps/persistent-storage/gs-setup.py

@@ -0,0 +1,32 @@
+import gspread
+import polars as pl
+
+# Authenticate with Google Sheets using a service account
+gc = gspread.service_account(filename="service_account.json")
+
+# Put your URL here
+sheet = gc.open_by_url("https://docs.google.com/spreadsheets/d/your_workbook_id")
+WORKSHEET = sheet.get_worksheet(0)
+
+import polars as pl
+
+# A polars schema that the data should conform to
+SCHEMA = {"name": pl.Utf8, "checkbox": pl.String, "favorite_number": pl.Int32}
+
+
+def load_data():
+    return pl.from_dicts(
+        WORKSHEET.get_all_records(expected_headers=SCHEMA.keys()), schema=SCHEMA
+    )
+
+
+def save_info(info: dict):
+    # Google Sheets expects a list of values for the new row
+    new_row = list(info.values())
+    WORKSHEET.append_row(new_row, insert_data_option="INSERT_ROWS")
+
+
+def append_info(d: pl.DataFrame, info: dict):
+    # Cast the boolean to a string for storage
+    info["checkbox"] = str(info["checkbox"])
+    return pl.concat([d, pl.DataFrame(info, schema=SCHEMA)], how="vertical")

docs/apps/persistent-storage/ibis-setup.py

@@ -0,0 +1,24 @@
+import ibis
+import polars as pl
+
+# NOTE:  app.py should import CONN and close it via
+# `_ = session.on_close(CONN.disconnect)` or similar
+CONN = ibis.postgres.connect(
+    user="postgres", password="", host="localhost", port=5432, database="template1"
+)
+TABLE_NAME = "testapp"
+
+SCHEMA = {"name": pl.Utf8, "checkbox": pl.Boolean, "favorite_number": pl.Int32}
+
+
+def load_data():
+    return CONN.table(TABLE_NAME).to_polars()
+
+
+def save_info(info: dict):
+    new_row = pl.DataFrame(info, schema=SCHEMA)
+    CONN.insert(TABLE_NAME, new_row, overwrite=False)
+
+
+def append_info(d: pl.DataFrame, info: dict):
+    return pl.concat([d, pl.DataFrame(info, schema=SCHEMA)], how="vertical")

docs/apps/persistent-storage/pg-setup.py

@@ -0,0 +1,19 @@
+import polars as pl
+
+URI = "postgresql://postgres@localhost:5432/template1"
+TABLE_NAME = "testapp"
+
+SCHEMA = {"name": pl.Utf8, "checkbox": pl.Boolean, "favorite_number": pl.Int32}
+
+
+def load_data():
+    return pl.read_database_uri(f"SELECT * FROM {TABLE_NAME}", URI)
+
+
+def save_info(info: dict):
+    new_row = pl.DataFrame(info, schema=SCHEMA)
+    new_row.write_database(TABLE_NAME, URI, if_table_exists="append")
+
+
+def append_info(d: pl.DataFrame, info: dict):
+    return pl.concat([d, pl.DataFrame(info, schema=SCHEMA)], how="vertical")

docs/apps/persistent-storage/s3-setup.py

@@ -0,0 +1,36 @@
+from datetime import datetime
+
+import polars as pl
+
+DATA_BUCKET = "s3://my-bucket/data/"
+STORAGE_OPTIONS = {
+    "aws_access_key_id": "<secret>",
+    "aws_secret_access_key": "<secret>",
+    "aws_region": "us-east-1",
+}
+
+SCHEMA = {
+    "name": pl.Utf8,
+    "checkbox": pl.String,
+    "favorite_number": pl.Int32,
+    "date": pl.Datetime,
+}
+
+
+def load_data():
+    return pl.read_parquet(
+        f"{DATA_BUCKET}**/*.parquet", storage_options=STORAGE_OPTIONS
+    )
+
+
+def save_info(info: dict):
+    info["date"] = datetime.now()
+    new_row = pl.DataFrame(info, schema=SCHEMA)
+    new_row.write_parquet(
+        f"{DATA_BUCKET}", partition_by="date", storage_options=STORAGE_OPTIONS
+    )
+
+
+def append_info(d: pl.DataFrame, info: dict):
+    info["date"] = datetime.now()
+    return pl.concat([d, pl.DataFrame(info, schema=SCHEMA)], how="vertical")

docs/apps/persistent-storage/simple_form_app.jpeg

@@ -0,0 +1,36 @@
+import ibis
+from ibis import _
+from shiny.express import ui, render, input, session
+from shiny import reactive
+
+# Connect to the database (quick, doesn't load data)
+con = ibis.postgres.connect(
+    user="", password="", host="", port=, database=""
+)
+dat = con.table("weather")
+end_session = session.on_ended(con.disconnect)
+
+with ui.sidebar():
+    ui.input_checkbox_group(
+        "season",
+        "Season",
+        choices=["Summer", "Winter", "Autumn", "Spring"],
+        selected="Summer",
+    )
+    # Import just the unique city names for our selectize input
+    cities = dat.select("city_name").distinct().execute()["city_name"].to_list()
+    ui.input_selectize("city", "City", choices=cities)
+
+
+# Store data manipulations in a reactive calculation
+# (convenient when using the data in multiple places)
+@reactive.calc
+def filtered_dat():
+    return dat.filter(
+        [_.city_name == input.city(), _.season.isin(input.season())]
+    )
+
+# Display the filtered data
+@render.data_frame
+def results_df():
+    return filtered_dat().execute()

docs/apps/reading-data/ibis-app.py

@@ -0,0 +1,32 @@
+import polars as pl
+from shiny import reactive
+from shiny.express import input, render, ui
+
+# Use `scan_*` instead of `read_*` to use the lazy API
+dat = pl.scan_parquet("./daily_weather.parquet")
+
+with ui.sidebar():
+    ui.input_checkbox_group(
+        "season",
+        "Season",
+        choices=["Summer", "Winter", "Fall", "Spring"],
+        selected="Summer",
+    )
+    # Import just the unique city names for our selectize input
+    cities = dat.select("city_name").unique().collect().to_series().to_list()
+    ui.input_selectize("city", "City", choices=cities)
+
+
+# Store manipulation in a reactive calc
+# (convenient for writing once and using in multiple places)
+@reactive.calc
+def filtered_dat():
+    return dat.filter(pl.col("city_name") == input.city()).filter(
+        pl.col("season").is_in(input.season())
+    )
+
+
+# Display the filtered data
+@render.data_frame
+def results_df():
+    return filtered_dat().collect()

docs/apps/reading-data/polars-app.py

@@ -0,0 +1,18 @@
+import ibis
+from shiny.express import render
+from shiny import reactive
+
+con = ibis.postgres.connect(user="", password="", host="", port=, database="")
+table = con.table("tablename")
+
+def check_last_updated():
+    return table.last_updated.max().execute()
+
+# Every 5 seconds, check if the max timestamp has changed
+@reactive.poll(check_last_updated, interval_secs=5)
+def data():
+    return table.execute()
+
+@render.data_frame
+def result():
+    return data()