Skip to content

Commit

Permalink
Interactive Brokers statement import: internal cash transfers import …
Browse files Browse the repository at this point in the history 
…implementation
  • Loading branch information
@titov-vv
titov-vv committed Apr 7, 2024
1 parent 23aab10 commit 892a169
Show file tree
Hide file tree
Showing 5 changed files with 37 additions and 18 deletions.
44 changes: 30 additions & 14 deletions jal/data_import/broker_statements/ibkr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,11 +284,13 @@ def __init__(self):
('account', 'account2', IBKR_Account, 0),
('dateTime', 'timestamp', datetime, None),
('quantity', 'quantity', float, None),
('type', 'description', str, None),
('cashTransfer', 'amount', float, None),
('type', 'type', str, None),
('description', 'description', str, None),
('direction', 'direction', str, None),
('company', 'company', str, None),
('transactionID', 'number', str, '')],
'loader': self.load_asset_transfers}
'loader': self.load_transfers}
}

@staticmethod
Expand Down Expand Up @@ -368,6 +370,10 @@ def attr_asset(self, xml_element, attr_name, default_value):
asset_id = [self.currency_id(code) for code in currency]
if not asset_id:
return default_value
elif xml_element.tag == 'Transfer' and asset_category == FOF.ASSET_MONEY:
asset_id = self.currency_id(xml_element.attrib['currency'])
if not asset_id:
return default_value
else:
symbol = xml_element.attrib[attr_name]
if symbol.endswith('.OLD'):
Expand Down Expand Up @@ -478,7 +484,7 @@ def load_ib_trades(self, ib_trades):
trades_loaded = self.load_trades(trades)

transfers = [transfer for transfer in ib_trades if type(transfer['asset']) == list]
transfers_loaded = self.load_transfers(transfers)
transfers_loaded = self.load_cash_deposits_withdrawals(transfers)

logging.info(self.tr("Trades loaded: ") + f"{trades_loaded + transfers_loaded} ({len(ib_trades)})")

Expand All @@ -502,7 +508,7 @@ def load_trades(self, trades):
cnt += 1
return cnt

def load_transfers(self, transfers):
def load_cash_deposits_withdrawals(self, transfers):
transfer_base = max([0] + [x['id'] for x in self._data[FOF.TRANSFERS]]) + 1
cnt = 0
for i, transfer in enumerate(sorted(transfers, key=lambda x: x['timestamp'])):
Expand All @@ -520,19 +526,29 @@ def load_transfers(self, transfers):
cnt += 1
return cnt

def load_asset_transfers(self, transfers):
def load_transfers(self, transfers):
transfer_base = max([0] + [x['id'] for x in self._data[FOF.TRANSFERS]]) + 1
cnt = 0
for i, transfer in enumerate(transfers):
transfer['id'] = transfer_base + i
if transfer['direction'] != "IN":
raise Statement_ImportError(self.tr("Outgoing asset transfer not implemented yet"))
transfer['account'] = [transfer['account2'], transfer['account'], 0]
transfer['asset'] = [transfer['asset'], transfer['asset']]
transfer['withdrawal'] = transfer['deposit'] = transfer.pop('quantity')
transfer['description'] = transfer['description'] + f" TRANSFER ({transfer.pop('company')})"
transfer['fee'] = 0.0
self.drop_extra_fields(transfer, ["direction", "account2"])
if self._find_in_list(self._data[FOF.ASSETS], 'id', transfer['asset'])['type'] == FOF.ASSET_MONEY:
if transfer['direction'] != "OUT":
raise Statement_ImportError(self.tr("Incoming money transfer not implemented yet"))
transfer['account'] = [transfer['account'], transfer.pop('account2'), 0]
transfer['asset'] = [transfer['asset'], transfer['asset']]
transfer['withdrawal'] = transfer['deposit'] = -transfer.pop('amount')
transfer['description'] = transfer.pop('type') + ' ' + transfer['description']
transfer['fee'] = 0.0
self.drop_extra_fields(transfer, ["direction", "amount", "company", "quantity"])
else:
if transfer['direction'] != "IN":
raise Statement_ImportError(self.tr("Outgoing asset transfer not implemented yet"))
transfer['account'] = [transfer.pop('account2'), transfer['account'], 0]
transfer['asset'] = [transfer['asset'], transfer['asset']]
transfer['withdrawal'] = transfer['deposit'] = transfer.pop('quantity')
transfer['description'] = transfer.pop('type') + f" TRANSFER ({transfer.pop('company')})"
transfer['fee'] = 0.0
self.drop_extra_fields(transfer, ["direction", "amount"])
self._data[FOF.TRANSFERS].append(transfer)
cnt += 1
return cnt
Expand Down Expand Up @@ -748,7 +764,7 @@ def load_symbol_change(self, action, parts_b) -> int:
raise Statement_ImportError(self.tr("Can't parse Symbol Change description ") + f"'{action}'")
isin_change = parts.groupdict()
if len(isin_change) != SymbolChangePattern.count("(?P<"): # check that expected number of groups was matched
raise Statement_ImportError(self.tr("Spin-off description miss some data ") + f"'{action}'")
raise Statement_ImportError(self.tr("Symbol Change description miss some data ") + f"'{action}'")
description_b = action['description'][:parts.span('symbol')[0]] + isin_change['symbol_old']
asset_b = self.locate_asset(isin_change['symbol_old'], isin_change['isin_old'])
paired_record = self.find_corp_action_pair(asset_b, description_b, action, parts_b)
Expand Down
3 changes: 2 additions & 1 deletion tests/test_data/ibkr.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,8 @@
{"id": 5, "account": [2, 0, 0], "number": "14333901913", "asset": [2, 2], "timestamp": 1605744000, "withdrawal": 1234.0, "deposit": 1234.0, "fee": 0.0, "description": "DISBURSEMENT INITIATED BY John Doe"},
{"id": 6, "account": [0, 2, 0], "number": "1234567890", "asset": [2, 2], "timestamp": 1663372800, "withdrawal": 100.0, "deposit": 100.0, "description": "CASH RECEIPTS / ELECTRONIC FUND TRANSFERS", "fee": 0.0},
{"id": 7, "account": [0, 2, 0], "number": "1234567891", "asset": [2, 2], "timestamp": 1663372800, "withdrawal": 100.0, "deposit": 100.0, "description": "CASH RECEIPTS / ELECTRONIC FUND TRANSFERS", "fee": 0.0},
{"id": 8, "account": [3, 2, 0], "number": "24055511103", "asset": [4 ,4], "timestamp": 1685702720, "withdrawal": 7.0, "deposit": 7.0, "description": "INTERNAL TRANSFER (--)", "fee": 0.0}
{"id": 8, "account": [3, 2, 0], "number": "24055511103", "asset": [4 ,4], "timestamp": 1685702720, "withdrawal": 7.0, "deposit": 7.0, "description": "INTERNAL TRANSFER (--)", "fee": 0.0},
{"id": 9, "account": [2, 3, 0], "number": "21632131212", "asset": [2, 2], "timestamp": 1694777165, "withdrawal": 12345.0, "deposit": 12345.0, "description": "INTERNAL TRANSFER FROM U7654321 TO TEST_ACC","fee": 0.0}
],
"corporate_actions": [
{"id": 1, "type": "split", "account": 2, "timestamp": 1618345500, "number": "16074977038", "asset": 4, "quantity": 217.0,
Expand Down
1 change: 1 addition & 0 deletions tests/test_data/ibkr.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@
</StockGrantActivities>
<Transfers>
<Transfer accountId="U7654321" acctAlias="" model="" currency="USD" fxRateToBase="1" assetCategory="STK" symbol="AMZN" description="AMAZON.COM INC" conid="3691937" securityID="US0231351067" securityIDType="ISIN" cusip="023135106" isin="US0231351067" listingExchange="NASDAQ" underlyingConid="" underlyingSymbol="" underlyingSecurityID="" underlyingListingExchange="" issuer="" multiplier="1" strike="" expiry="" putCall="" principalAdjustFactor="" reportDate="20230602" date="20230602" dateTime="20230602;104520" type="INTERNAL" direction="IN" company="--" account="TEST_ACC" accountName="" quantity="7" transferPrice="0" positionAmount="1234.56" positionAmountInBase="1234.56" pnlAmount="0" pnlAmountInBase="0" fxPnl="0" cashTransfer="0" code="" clientReference="" transactionID="24055511103" />
<Transfer accountId="U7654321" acctAlias="" model="" currency="USD" fxRateToBase="1" assetCategory="CASH" subCategory="" symbol="--" description="TRANSFER FROM U7654321 TO TEST_ACC" conid="" securityID="" securityIDType="" cusip="" isin="" figi="" listingExchange="" underlyingConid="" underlyingSymbol="" underlyingSecurityID="" underlyingListingExchange="" issuer="" issuerCountryCode="" multiplier="0" strike="" expiry="" putCall="" principalAdjustFactor="" reportDate="20230915" date="20230915" dateTime="20230915;112605" type="INTERNAL" direction="OUT" company="--" account="TEST_ACC" accountName="-" deliveringBroker="" quantity="0" transferPrice="0" positionAmount="0" positionAmountInBase="0" pnlAmount="0" pnlAmountInBase="0" cashTransfer="-12345" code="" clientReference="" transactionID="21632131212" levelOfDetail="TRANSFER" serialNumber="" deliveryType="" commodityType="" fineness="0.0" weight="0.0" />
</Transfers>
</FlexStatement>
</FlexStatements>
Expand Down
3 changes: 2 additions & 1 deletion tests/test_data/matched.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,8 @@
{"id": 5, "account": [-1, 0, 0], "number": "14333901913", "asset": [-2, -2], "timestamp": 1605744000, "withdrawal": 1234.0, "deposit": 1234.0, "fee": 0.0, "description": "DISBURSEMENT INITIATED BY John Doe"},
{"id": 6, "account": [0, -1, 0], "number": "1234567890", "asset": [-2, -2], "timestamp": 1663372800, "withdrawal": 100.0, "deposit": 100.0, "description": "CASH RECEIPTS / ELECTRONIC FUND TRANSFERS", "fee": 0.0},
{"id": 7, "account": [0, -1, 0], "number": "1234567891", "asset": [-2, -2], "timestamp": 1663372800, "withdrawal": 100.0, "deposit": 100.0, "description": "CASH RECEIPTS / ELECTRONIC FUND TRANSFERS", "fee": 0.0},
{"id": 8, "account": [3, -1, 0], "number": "24055511103", "asset": [4 ,4], "timestamp": 1685702720, "withdrawal": 7.0, "deposit": 7.0, "description": "INTERNAL TRANSFER (--)", "fee": 0.0}
{"id": 8, "account": [3, -1, 0], "number": "24055511103", "asset": [4 ,4], "timestamp": 1685702720, "withdrawal": 7.0, "deposit": 7.0, "description": "INTERNAL TRANSFER (--)", "fee": 0.0},
{"id": 9, "account": [-1, 3, 0], "number": "21632131212", "asset": [-2, -2], "timestamp": 1694777165, "withdrawal": 12345.0, "deposit": 12345.0, "description": "INTERNAL TRANSFER FROM U7654321 TO TEST_ACC", "fee": 0.0}
],
"corporate_actions": [
{"id": 1, "type": "split", "account": -1, "timestamp": 1618345500, "number": "16074977038", "asset": 4, "quantity": 217.0,
Expand Down
4 changes: 2 additions & 2 deletions tests/test_json_import.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from jal.constants import PredefinedAsset
from jal.db.account import JalAccount
from jal.db.asset import JalAsset, AssetData
from jal.db.peer import JalPeer


def test_ibkr_json_import(tmp_path, project_root, data_path, prepare_db_ibkr):
Expand Down Expand Up @@ -184,7 +183,8 @@ def test_ibkr_json_import(tmp_path, project_root, data_path, prepare_db_ibkr):
[5, 4, 1605744000, 1, '1234.0', 1605744000, 1, '1234.0', '', '', '14333901913', '', 'DISBURSEMENT INITIATED BY John Doe'],
[6, 4, 1663372800, 1, '100.0', 1663372800, 1, '100.0', '', '', '1234567890', '', 'CASH RECEIPTS / ELECTRONIC FUND TRANSFERS'],
[7, 4, 1663372800, 1, '100.0', 1663372800, 1, '100.0', '', '', '1234567891', '', 'CASH RECEIPTS / ELECTRONIC FUND TRANSFERS'],
[8, 4, 1685702720, 3, '7.0', 1685702720, 1, '7.0', '', '', '24055511103', 8, 'INTERNAL TRANSFER (--)']
[8, 4, 1685702720, 3, '7.0', 1685702720, 1, '7.0', '', '', '24055511103', 8, 'INTERNAL TRANSFER (--)'],
[9, 4, 1694777165, 1, '12345.0', 1694777165, 3, '12345.0', '', '', '21632131212', '', 'INTERNAL TRANSFER FROM U7654321 TO TEST_ACC']
]

# validate trades
Expand Down

0 comments on commit 892a169

Please sign in to comment.