Skip to content

Commit d13b787

Browse files
krkeegankrkeegan
authored
Merge pull request #531 from tstabrawa/db-2nd-pass
If not all DB records are received, try one-by-one
2 parents 7d2bb90 + c192ac6 commit d13b787

File tree

8 files changed

+708
-15
lines changed

8 files changed

+708
-15
lines changed

‎insteon_mqtt/db/Device.py

+18-1
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,23 @@ def is_current(self, delta):
174174
"""
175175
return delta == self.delta
176176

177+
#-----------------------------------------------------------------------
178+
def is_complete(self):
179+
"""See if the database is complete.
180+
181+
Check that all DB entries between START_MEM_LOC and self.last
182+
(inclusive) are present in either self.entries or self.unused.
183+
184+
Returns:
185+
(bool) Returns True if the database is complete.
186+
"""
187+
if (self.last not in self.entries.values() and
188+
self.last not in self.unused.values()):
189+
# Last entry hasn't been added/downloaded yet
190+
return False
191+
expected_entries = ((START_MEM_LOC - self.last.mem_loc) / 8) + 1
192+
return len(self.entries) + len(self.unused) == expected_entries
193+
177194
#-----------------------------------------------------------------------
178195
def increment_delta(self):
179196
"""Increments the current database delta by 1
@@ -770,7 +787,7 @@ def _add_using_unused(self, addr, group, is_controller, data,
770787
#-----------------------------------------------------------------------
771788
def _add_using_new(self, addr, group, is_controller, data,
772789
on_done):
773-
"""Add a anew entry at the end of the database.
790+
"""Add a new entry at the end of the database.
774791
775792
First we send the new entry to the remote device. If that works,
776793
then we update the previously last entry to mart it as "not the last

‎insteon_mqtt/db/DeviceScanManagerI2.py

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
#===========================================================================
2+
#
3+
# Device Scan Manager for i2 or newer Devices
4+
#
5+
#===========================================================================
6+
from .. import log
7+
from .. import message as Msg
8+
from .. import util
9+
from .. import handler
10+
from .DeviceEntry import DeviceEntry
11+
from .Device import START_MEM_LOC
12+
13+
LOG = log.get_logger()
14+
15+
16+
class DeviceScanManagerI2:
17+
"""Manager for scaning the link database of an i2 or newer device.
18+
19+
This class can be used to download any entries that failed to download
20+
using DeviceDbGet.py (or all entries, if started with a cleared DB).
21+
"""
22+
def __init__(self, device, device_db, on_done=None, *, num_retry=3,
23+
mem_addr: int = START_MEM_LOC):
24+
"""Constructor
25+
26+
Args
27+
device: (Device) The Insteon Device object
28+
device_db: (db.Device) The device database being retrieved.
29+
on_done: Finished callback. Will be called when the scan
30+
operation is done.
31+
Keyword-only Args:
32+
num_retry: (int) The number of times to retry each message if the
33+
handler times out without returning Msg.FINISHED.
34+
This count does include the initial sending so a
35+
retry of 3 will send once and then retry 2 more times.
36+
mem_addr: (int) Address at which to start downloading.
37+
"""
38+
self.db = device_db
39+
self.device = device
40+
self._mem_addr = mem_addr
41+
self.on_done = util.make_callback(on_done)
42+
self._num_retry = num_retry
43+
44+
#-------------------------------------------------------------------
45+
def start_scan(self):
46+
"""Start a managed scan of a i2 (or newer) device database."""
47+
self._request_next_record(self.on_done)
48+
49+
#-------------------------------------------------------------------
50+
def _request_next_record(self, on_done):
51+
"""Request the next missing DB record.
52+
53+
Args:
54+
on_done: (callback) a callback that is passed around and run on the
55+
completion of the scan
56+
"""
57+
58+
done, last_entry = self._calculate_next_addr()
59+
if done:
60+
if self.db.is_complete():
61+
on_done(True, "Database received", last_entry)
62+
else:
63+
on_done(False, "Database incomplete", last_entry)
64+
return
65+
66+
data = bytes([
67+
0x00,
68+
0x00, # ALDB record request
69+
self._mem_addr >> 8, # Address MSB
70+
self._mem_addr & 0xff, # Address LSB
71+
0x01, # Read one record
72+
] + [0x00] * 9)
73+
msg = Msg.OutExtended.direct(self.device.addr, 0x2f, 0x00, data)
74+
msg_handler = handler.ExtendedCmdResponse(msg, self.handle_record,
75+
on_done=on_done,
76+
num_retry=self._num_retry)
77+
self.device.send(msg, msg_handler)
78+
79+
#-------------------------------------------------------------------
80+
def handle_record(self, msg, on_done):
81+
"""Handle an ALDB record response by adding an entry to the DB and
82+
fetching the next entry.
83+
84+
Args:
85+
msg: (message.InpExtended) The ALDB record response.
86+
on_done: (callback) a callback that is passed around and run on the
87+
completion of the scan
88+
"""
89+
90+
# Convert the message to a database device entry.
91+
entry = DeviceEntry.from_bytes(msg.data, db=self.db)
92+
LOG.ui("Entry: %s", entry)
93+
94+
# Skip entries w/ a null memory location.
95+
if entry.mem_loc:
96+
self.db.add_entry(entry)
97+
98+
self._request_next_record(on_done)
99+
100+
#-------------------------------------------------------------------
101+
def _calculate_next_addr(self) -> (bool, DeviceEntry):
102+
"""Calculate the memory address of the next missing record.
103+
104+
Returns:
105+
(bool) True if no more records to read
106+
(DeviceEntry) Last entry or closest-to-last (if not received yet)
107+
"""
108+
done = False
109+
last = None
110+
addr = self._mem_addr
111+
entry = self.db.entries.get(addr, self.db.unused.get(addr, None))
112+
while entry is not None:
113+
last = entry
114+
if self.db.last.identical(entry):
115+
# This is the last record (and we already have it)
116+
done = True
117+
break
118+
addr -= 0x8
119+
entry = self.db.entries.get(addr, self.db.unused.get(addr, None))
120+
self._mem_addr = addr
121+
return done, last

‎insteon_mqtt/db/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,6 @@
1818
from .DeviceEntry import DeviceEntry
1919
from .DeviceModifyManagerI1 import DeviceModifyManagerI1
2020
from .DeviceScanManagerI1 import DeviceScanManagerI1
21+
from .DeviceScanManagerI2 import DeviceScanManagerI2
2122
from .Modem import Modem
2223
from .ModemEntry import ModemEntry

‎insteon_mqtt/handler/DeviceDbGet.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,10 @@ def msg_received(self, protocol, msg):
143143
# Note that if the entry is a null entry (all zeros), then
144144
# is_last_rec will be True as well.
145145
if entry.db_flags.is_last_rec:
146-
self.on_done(True, "Database received", entry)
146+
if self.db.is_complete():
147+
self.on_done(True, "Database received", entry)
148+
else:
149+
self.on_done(False, "Database incomplete", entry)
147150
return Msg.FINISHED
148151

149152
# Otherwise keep processing records as they arrive.

‎insteon_mqtt/handler/DeviceRefresh.py

+19-3
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ def msg_received(self, protocol, msg):
106106
# Clear the current database values.
107107
self.device.db.clear()
108108

109-
# When the update message below ends, update the db delta
109+
# When database download is complete, update the db delta
110110
# w/ the current value and save the database.
111111
def on_done(success, message, data):
112112
if success:
@@ -116,7 +116,22 @@ def on_done(success, message, data):
116116
self.addr, self.device.db)
117117
self.on_done(success, message, data)
118118

119-
# Request that the device send us all of it's database
119+
# Called after DeviceDbGet finishes trying a bulk download
120+
def on_done_dbget(success, message, data):
121+
if success:
122+
# Skip gap-filling step & call above-defined func
123+
on_done(success, message, data)
124+
else:
125+
LOG.warning("%s database bulk download error: %s",
126+
self.addr, message)
127+
# Try filling-in gaps one entry at a time
128+
manager = db.DeviceScanManagerI2(self.device,
129+
self.device.db,
130+
on_done=on_done,
131+
num_retry=3)
132+
manager.start_scan()
133+
134+
# Request that the device send us all of its database
120135
# records. These will be streamed as fast as possible to
121136
# us and the handler will update the database. We need a
122137
# retry count here because battery powered devices don't
@@ -130,7 +145,8 @@ def on_done(success, message, data):
130145
else:
131146
db_msg = Msg.OutExtended.direct(self.addr, 0x2f, 0x00,
132147
bytes(14))
133-
msg_handler = DeviceDbGet(self.device.db, on_done,
148+
msg_handler = DeviceDbGet(self.device.db,
149+
on_done_dbget,
134150
num_retry=3)
135151
self.device.send(db_msg, msg_handler)
136152
# Either way - this transaction is complete.

‎tests/db/test_DeviceScanManagerI2.py

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
#===========================================================================
2+
#
3+
# Tests for: insteont_mqtt/db/DeviceScanManagerI2.py
4+
#
5+
#===========================================================================
6+
import pytest
7+
import insteon_mqtt as IM
8+
import insteon_mqtt.message as Msg
9+
from insteon_mqtt.db.Device import START_MEM_LOC
10+
11+
class Test_DeviceScanManagerI2:
12+
def test_start_scan(self):
13+
dev_addr = IM.Address('0a.12.34')
14+
device = MockDevice(dev_addr, 0)
15+
manager = IM.db.DeviceScanManagerI2(device, device.db)
16+
17+
first_mem_addr = START_MEM_LOC
18+
data = bytes([
19+
0x00,
20+
0x00, # ALDB record request
21+
first_mem_addr >> 8, # Address MSB
22+
first_mem_addr & 0xff, # Address LSB
23+
0x01, # Read one record
24+
] + [0x00] * 9)
25+
db_msg = Msg.OutExtended.direct(dev_addr, 0x2f, 0x00, data)
26+
27+
manager.start_scan()
28+
assert device.sent[0].to_bytes() == db_msg.to_bytes()
29+
30+
#-------------------------------------------------------------------
31+
@pytest.mark.parametrize("init_db,mem_addr,recv,exp_calls,next_addr", [
32+
# Have all but last record. Receive it, making DB complete.
33+
( [ [ 0x00, 0x00, 0x0f, 0xff, 0xff,
34+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
35+
[ 0x00, 0x00, 0x0f, 0xf7, 0xff,
36+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
37+
[ 0x00, 0x00, 0x0f, 0xef, 0xff,
38+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ] ],
39+
None,
40+
[ 0x00, 0x01, 0x0f, 0xe7, 0xff,
41+
0x00, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
42+
[ "Database received" ],
43+
None ),
44+
# Have all but third record. Receive it, making DB complete.
45+
( [ [ 0x00, 0x00, 0x0f, 0xff, 0xff,
46+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
47+
[ 0x00, 0x00, 0x0f, 0xf7, 0xff,
48+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
49+
[ 0x00, 0x00, 0x0f, 0xe7, 0xff,
50+
0x00, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ] ],
51+
None,
52+
[ 0x00, 0x01, 0x0f, 0xef, 0xff,
53+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
54+
[ "Database received" ],
55+
None ),
56+
# Have first two records. Receive third and request fourth.
57+
( [ [ 0x00, 0x00, 0x0f, 0xff, 0xff,
58+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
59+
[ 0x00, 0x00, 0x0f, 0xef, 0xff,
60+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ] ],
61+
None,
62+
[ 0x00, 0x01, 0x0f, 0xf7, 0xff,
63+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
64+
[ ],
65+
0xfe7 ),
66+
# Have first two records. Receive last. DB still incomplete.
67+
( [ [ 0x00, 0x00, 0x0f, 0xff, 0xff,
68+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
69+
[ 0x00, 0x00, 0x0f, 0xf7, 0xff,
70+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ] ],
71+
None,
72+
[ 0x00, 0x01, 0x0f, 0xe7, 0xff,
73+
0x00, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
74+
[ "Database incomplete" ],
75+
None ),
76+
# Have first and last records. Receive third. DB still incomplete.
77+
( [ [ 0x00, 0x00, 0x0f, 0xff, 0xff,
78+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
79+
[ 0x00, 0x00, 0x0f, 0xe7, 0xff,
80+
0x00, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ] ],
81+
None,
82+
[ 0x00, 0x01, 0x0f, 0xef, 0xff,
83+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
84+
[ "Database incomplete" ],
85+
None ),
86+
# Have first and last records. Receive second and request third.
87+
( [ [ 0x00, 0x00, 0x0f, 0xff, 0xff,
88+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
89+
[ 0x00, 0x00, 0x0f, 0xe7, 0xff,
90+
0x00, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ] ],
91+
None,
92+
[ 0x00, 0x01, 0x0f, 0xf7, 0xff,
93+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
94+
[ ],
95+
0xfef ),
96+
# Have first and last records. Expecting second, but receive third.
97+
# Request second.
98+
( [ [ 0x00, 0x00, 0x0f, 0xff, 0xff,
99+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
100+
[ 0x00, 0x00, 0x0f, 0xe7, 0xff,
101+
0x00, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ] ],
102+
0xff7,
103+
[ 0x00, 0x01, 0x0f, 0xef, 0xff,
104+
0xff, 0x01, 0x0a, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00 ],
105+
[ ],
106+
0xff7 )
107+
])
108+
def test_handle_record(self, init_db, mem_addr, recv, exp_calls,
109+
next_addr):
110+
calls = []
111+
112+
def callback(success, msg, value):
113+
calls.append(msg)
114+
115+
modem_addr = IM.Address('09.12.34')
116+
dev_addr = IM.Address('0a.12.34')
117+
device = MockDevice(dev_addr, 0)
118+
if mem_addr is None:
119+
# Expecting the same address that we're about to receive
120+
mem_addr = (recv[2] << 8) + recv[3]
121+
manager = IM.db.DeviceScanManagerI2(device, device.db, callback,
122+
mem_addr=mem_addr)
123+
124+
# Initialize DB starting state
125+
for data in init_db:
126+
entry = IM.db.DeviceEntry.from_bytes(data, db=device.db)
127+
device.db.add_entry(entry)
128+
129+
# Handle received message, check callbacks
130+
flags = Msg.Flags(Msg.Flags.Type.DIRECT, True)
131+
msg = Msg.InpExtended(dev_addr, modem_addr, flags, 0x2f, 0x00, recv)
132+
manager.handle_record(msg, callback)
133+
assert len(calls) == len(exp_calls)
134+
for idx, call in enumerate(exp_calls):
135+
assert calls[idx] == call
136+
137+
# Check address of next requested entry (if any expected)
138+
if next_addr is not None:
139+
assert len(device.sent) == 1
140+
sent_data = device.sent[0].data
141+
requested_addr = (sent_data[2] << 8) + sent_data[3]
142+
assert requested_addr == next_addr
143+
144+
#===========================================================================
145+
class MockDevice:
146+
"""Mock insteon_mqtt/Device class
147+
"""
148+
def __init__(self, addr, db_delta):
149+
self.sent = []
150+
self.addr = addr
151+
self.db = IM.db.Device(addr, None, self)
152+
self.db.delta = db_delta
153+
154+
def send(self, msg, handler, priority=None, after=None):
155+
self.sent.append(msg)

0 commit comments

Comments
 (0)