Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion automation/config.example.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
"lookbackDays": 90,
"invertAmounts": false,
"accounts": [
{ "plaidAccountId": "acc_example", "deskbooksAccountId": 1 }
{ "plaidAccountId": "acc_example", "deskbooksAccountId": 1 },
{ "plaidAccountId": "acc_example_2", "deskbooksAccountId": 2, "balances": false }
]
},
{
Expand Down
23 changes: 21 additions & 2 deletions automation/fetchers/plaid.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,13 @@
* "lookbackDays": 90,
* "invertAmounts": false,
* "accounts": [
* { "plaidAccountId": "acc_...", "deskbooksAccountId": 3 }
* { "plaidAccountId": "acc_...", "deskbooksAccountId": 3 },
* { "plaidAccountId": "acc_...", "deskbooksAccountId": 9, "balances": false }
* ]
* }
*
* "balances": false stages the account's transactions but never its
* balance, keeping it out of the net-worth series.
*/
import { readFile } from "node:fs/promises";
import { httpsPostJson } from "../src/connector-http.mjs";
Expand Down Expand Up @@ -103,8 +107,16 @@ export function normalizePlaidBalances({ mappings, accountsById }) {
// Balances of provider accounts sharing a DeskBooks account are summed
// (integer-cent math). A row is emitted as null only when every mapped
// provider account reports a null balance.
//
// "balances": false opts a mapping out entirely: its transactions still
// import, but no balance row is ever staged, so the account stays out of
// the net-worth series (net worth is the sum of snapshot balance rows).
// Donor-advised funds are the motivating case — the giving is worth
// tracking, the balance is money you no longer own.
const rows = [];
for (const [deskbooksAccountId, plaidIds] of groupMappings(mappings)) {
for (const [deskbooksAccountId, plaidIds] of groupMappings(
mappings.filter((mapping) => mapping.balances !== false),
)) {
let cents = 0;
let seen = 0;
for (const plaidId of plaidIds) {
Expand Down Expand Up @@ -147,6 +159,13 @@ function validateSource(source) {
if (!mapping.plaidAccountId || !Number.isInteger(mapping.deskbooksAccountId)) {
throw new Error(`${source.name}: each account needs plaidAccountId and integer deskbooksAccountId`);
}
// Fail loud rather than silently staging a balance the mapping meant
// to suppress — a typo here quietly lands money in net worth.
if ("balances" in mapping && typeof mapping.balances !== "boolean") {
throw new Error(
`${source.name}: account ${mapping.plaidAccountId}: "balances" must be true or false, got: ${JSON.stringify(mapping.balances)}`,
);
}
}
return accounts;
}
Expand Down
24 changes: 24 additions & 0 deletions automation/tests/staged-formats.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,30 @@ test("normalizePlaidBalances sums provider accounts that share a DeskBooks accou
]);
});

test("normalizePlaidBalances skips mappings marked balances:false", () => {
const rows = normalizePlaidBalances({
mappings: [
{ plaidAccountId: "taxable", deskbooksAccountId: 3 },
{ plaidAccountId: "daf", deskbooksAccountId: 9, balances: false },
],
accountsById: {
taxable: { balances: { current: 100.25 } },
daf: { balances: { current: 5000 } },
},
});
// The DAF reports a balance and is still omitted entirely — not even a
// null row, which would read as "account did not exist yet".
assert.deepEqual(rows, [{ accountId: 3, balance: "100.25" }]);
});

test("normalizePlaidBalances keeps mappings that set balances:true", () => {
const rows = normalizePlaidBalances({
mappings: [{ plaidAccountId: "sav", deskbooksAccountId: 6, balances: true }],
accountsById: { sav: { balances: { current: 55.55 } } },
});
assert.deepEqual(rows, [{ accountId: 6, balance: "55.55" }]);
});

test("groupMappings folds many provider accounts into one DeskBooks account", () => {
const grouped = groupMappings([
{ plaidAccountId: "a", deskbooksAccountId: 1 },
Expand Down
26 changes: 23 additions & 3 deletions backend/app/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,19 +43,38 @@ def _active_db_path() -> Path:
("fire_settings", "growth_property", "NUMERIC NOT NULL DEFAULT 0.0100"),
("transactions", "budget_date", "DATE"),
("transactions", "kind_before_pair", "VARCHAR"),
("rules", "set_is_excluded_from_totals", "BOOLEAN"),
)


def _apply_additive_columns(engine: Engine) -> None:
with engine.begin() as conn:
for table, column, ddl in _ADDITIVE_COLUMNS:
existing = {
row[1] for row in conn.exec_driver_sql(f"PRAGMA table_info({table})")
}
existing = {row[1] for row in conn.exec_driver_sql(f"PRAGMA table_info({table})")}
if existing and column not in existing:
conn.exec_driver_sql(f"ALTER TABLE {table} ADD COLUMN {column} {ddl}")


# The mirror image: columns dropped from the models. create_all never
# removes anything, so a database created before the removal keeps them —
# and a leftover NOT NULL column with no default makes every INSERT into
# that table fail, since nothing supplies a value any more. Databases
# created after the removal never had the column, so this is a no-op for
# them. Only list columns no model or query references.
_DROPPED_COLUMNS: tuple[tuple[str, str], ...] = (
("accounts", "is_liquid"),
("accounts", "is_taxable"),
)


def _drop_removed_columns(engine: Engine) -> None:
with engine.begin() as conn:
for table, column in _DROPPED_COLUMNS:
existing = {row[1] for row in conn.exec_driver_sql(f"PRAGMA table_info({table})")}
if column in existing:
conn.exec_driver_sql(f"ALTER TABLE {table} DROP COLUMN {column}")


def engine_for(db_path: Path) -> Engine:
"""One cached engine per database file; tables ensured on first use."""
from . import models # noqa: F401 ensure models are imported
Expand All @@ -76,6 +95,7 @@ def engine_for(db_path: Path) -> Engine:
# the engine before its tables exist.
models.Base.metadata.create_all(bind=engine)
_apply_additive_columns(engine)
_drop_removed_columns(engine)
_engines[db_path] = engine
_factories[db_path] = sessionmaker(
bind=engine,
Expand Down
5 changes: 5 additions & 0 deletions backend/app/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,11 @@ class Rule(Base):
)
set_merchant: Mapped[str | None] = mapped_column(String(255))
set_tags: Mapped[list | None] = mapped_column(JSON)
# Stamps matched rows out of every total while leaving them in the
# ledger. The motivating case is an account you hold for the record
# but never want counted — a donor-advised fund, where the giving was
# already counted on the way in and the grants out would double it.
set_is_excluded_from_totals: Mapped[bool | None] = mapped_column(Boolean)
notes: Mapped[str | None] = mapped_column(Text)
last_applied_at: Mapped[datetime | None] = mapped_column(DateTime)
apply_count: Mapped[int] = mapped_column(Integer, default=0)
Expand Down
3 changes: 3 additions & 0 deletions backend/app/routers/imports.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,8 @@ def _preview_from_bytes(
r.merchant = ev.merchant
if ev.tags:
r.suggested_tags = ev.tags
if ev.is_excluded_from_totals is not None:
r.suggested_is_excluded_from_totals = ev.is_excluded_from_totals
if ev.matched_rule_id:
r.suggested_matched_rule_id = ev.matched_rule_id

Expand Down Expand Up @@ -276,6 +278,7 @@ def apply(body: schemas.ImportApplyRequest, db: DbSession):
amount=r.amount,
category_id=r.suggested_category_id,
kind=r.suggested_kind,
is_excluded_from_totals=r.suggested_is_excluded_from_totals,
is_user_categorized=False,
matched_rule_id=r.suggested_matched_rule_id,
import_batch_id=batch.id,
Expand Down
8 changes: 8 additions & 0 deletions backend/app/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ class RuleEval:
kind: models.TransactionKind | None = None
merchant: str | None = None
tags: list[str] | None = None
is_excluded_from_totals: bool | None = None
matched_rule_id: int | None = None


Expand Down Expand Up @@ -102,6 +103,7 @@ def evaluate(
kind=r.set_kind,
merchant=r.set_merchant,
tags=list(r.set_tags) if r.set_tags else None,
is_excluded_from_totals=r.set_is_excluded_from_totals,
matched_rule_id=r.id,
)
return RuleEval()
Expand Down Expand Up @@ -139,6 +141,12 @@ def reapply_to_unreviewed(db: Session) -> tuple[int, int]:
if ev.merchant and tx.merchant != ev.merchant:
tx.merchant = ev.merchant
changed = True
if (
ev.is_excluded_from_totals is not None
and tx.is_excluded_from_totals != ev.is_excluded_from_totals
):
tx.is_excluded_from_totals = ev.is_excluded_from_totals
changed = True
if changed and ev.matched_rule_id is not None:
tx.matched_rule_id = ev.matched_rule_id
fires[ev.matched_rule_id] += 1
Expand Down
4 changes: 4 additions & 0 deletions backend/app/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ class RuleIn(BaseModel):
set_kind: TransactionKind | None = None
set_merchant: str | None = None
set_tags: list[str] | None = None
set_is_excluded_from_totals: bool | None = None
notes: str | None = None


Expand All @@ -316,6 +317,7 @@ class RuleUpdate(BaseModel):
set_kind: TransactionKind | None = None
set_merchant: str | None = None
set_tags: list[str] | None = None
set_is_excluded_from_totals: bool | None = None
notes: str | None = None


Expand All @@ -336,6 +338,7 @@ class RuleOut(ORMBase):
set_kind: TransactionKind | None
set_merchant: str | None
set_tags: list[str] | None
set_is_excluded_from_totals: bool | None
notes: str | None
apply_count: int
last_applied_at: datetime | None
Expand Down Expand Up @@ -595,6 +598,7 @@ class ImportDraftRow(BaseModel):
suggested_category_id: int | None = None
suggested_kind: TransactionKind = TransactionKind.uncategorized
suggested_tags: list[str] = []
suggested_is_excluded_from_totals: bool = False
suggested_matched_rule_id: int | None = None
is_duplicate: bool = False
raw: dict | None = None
Expand Down
132 changes: 132 additions & 0 deletions backend/tests/test_rule_exclude_from_totals.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
"""A rule can stamp matched rows out of every total.

The motivating case is a donor-advised fund: the giving is already
counted as a donation on the way in, from the account that funded it, so
the fund's own rows — the mirrored contribution and the grants out — must
stay visible in the ledger without being counted a second time.
"""

from __future__ import annotations

from datetime import date
from decimal import Decimal

from app import analytics
from app.models import (
Account,
AccountCategory,
AccountType,
Category,
CategoryKind,
Rule,
SignConvention,
Transaction,
TransactionKind,
)
from app.rules import evaluate, load_active_rules, reapply_to_unreviewed


def _account(db, name: str) -> Account:
account = Account(
name=name,
account_category=AccountCategory.investment,
type=AccountType.brokerage,
sign_convention=SignConvention.outflow_negative,
)
db.add(account)
db.flush()
return account


def _transaction(db, account: Account, description: str, amount: str, **kwargs) -> Transaction:
tx = Transaction(
account_id=account.id,
date=date(2026, 3, 23),
description_raw=description,
amount=Decimal(amount),
kind=kwargs.pop("kind", TransactionKind.uncategorized),
is_user_categorized=kwargs.pop("is_user_categorized", False),
is_excluded_from_totals=kwargs.pop("is_excluded_from_totals", False),
**kwargs,
)
db.add(tx)
db.flush()
return tx


def _exclude_account_rule(db, account: Account) -> Rule:
rule = Rule(
name=f"{account.name}: informational only",
match_account_id=account.id,
set_is_excluded_from_totals=True,
)
db.add(rule)
db.flush()
return rule


def test_evaluate_returns_the_exclusion_for_a_matching_account(db):
fund = _account(db, "Giving Fund")
other = _account(db, "Taxable")
_exclude_account_rule(db, fund)
rules = load_active_rules(db)

matched = evaluate(rules, account_id=fund.id, description="GRANT", amount=Decimal("-500"))
assert matched.is_excluded_from_totals is True

unmatched = evaluate(rules, account_id=other.id, description="GRANT", amount=Decimal("-500"))
assert unmatched.is_excluded_from_totals is None


def test_rules_without_the_action_leave_the_flag_alone(db):
account = _account(db, "Taxable")
db.add(Rule(name="tag it", match_account_id=account.id, set_merchant="Somebody"))
db.flush()

ev = evaluate(load_active_rules(db), account_id=account.id, description="X", amount=Decimal("-1"))
assert ev.is_excluded_from_totals is None


def test_reapply_excludes_existing_unreviewed_rows(db):
fund = _account(db, "Giving Fund")
grant = _transaction(db, fund, "GRANT TO A CHARITY", "-500")
_exclude_account_rule(db, fund)

rows_changed, _ = reapply_to_unreviewed(db)

assert rows_changed == 1
assert grant.is_excluded_from_totals is True


def test_sankey_counts_the_contribution_and_ignores_the_fund(db):
"""The donation lands on the contribution date, once."""
taxable = _account(db, "Taxable")
fund = _account(db, "Giving Fund")
giving = Category(name="Giving", kind=CategoryKind.expense)
db.add(giving)
db.flush()

# The contribution out of the taxable account: this is the donation.
_transaction(
db,
taxable,
"Contribution to the fund",
"-1000",
kind=TransactionKind.donation,
category_id=giving.id,
is_user_categorized=True,
)
# The fund's own mirrored inflow and a later grant out.
_transaction(db, fund, "Contribution received", "1000", kind=TransactionKind.income)
_transaction(db, fund, "Grant to a charity", "-400", kind=TransactionKind.donation)
_exclude_account_rule(db, fund)
reapply_to_unreviewed(db)
db.commit()

result = analytics.cashflow_sankey(db, date(2026, 1, 1), date(2026, 12, 31), "2026")
links = {link["label"]: link["value"] for link in result["links"]}

# 1000 counted once, on the contribution date — not 1400 (contribution
# plus grant), and not 600 (contribution netted against the fund's
# mirrored inflow).
assert links["Donations"] == 1000.0
Loading
Loading