Skip to content

Commit

Permalink
improve limit tests with parallelized case
Browse files Browse the repository at this point in the history
  • Loading branch information
sh-rp committed Dec 16, 2024
1 parent f109a87 commit caf92bf
Showing 1 changed file with 12 additions and 6 deletions.
18 changes: 12 additions & 6 deletions tests/extract/test_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -839,26 +839,32 @@ def test_limit_infinite_counter() -> None:

@pytest.mark.parametrize("limit", (None, -1, 0, 10))
def test_limit_edge_cases(limit: int) -> None:
r = dlt.resource(range(20), name="infinity").add_limit(limit) # type: ignore
r = dlt.resource(range(20), name="resource").add_limit(limit) # type: ignore

@dlt.resource()
async def r_async():
for i in range(20):
await asyncio.sleep(0.01)
yield i

@dlt.resource(parallelized=True)
def parallelized_resource():
for i in range(20):
yield i

sync_list = list(r)
async_list = list(r_async().add_limit(limit))
parallelized_list = list(parallelized_resource().add_limit(limit))

# all lists should be the same
assert sync_list == async_list == parallelized_list

if limit == 10:
assert sync_list == list(range(10))
# we have edge cases where the async list will have one extra item
# possibly due to timing issues, maybe some other implementation problem
assert (async_list == list(range(10))) or (async_list == list(range(11)))
elif limit in [None, -1]:
assert sync_list == async_list == list(range(20))
assert sync_list == list(range(20))
elif limit == 0:
assert sync_list == async_list == []
assert sync_list == []
else:
raise AssertionError(f"Unexpected limit: {limit}")

Expand Down

0 comments on commit caf92bf

Please sign in to comment.