from_schema triggers filter_too_much on a simple schema

[aplainzetakind]

I have a straightforward and fairly small json schema (about 20-30 fields that are booleans or strings, with no custom formats), and calling from_schema on that raises the filter_too_much check. What causes this and is there anything to be done to improve performance? How is something invalid generated in the first place with these fields? One output says 8 valid and 42 invalid examples were generated.

[Zac-HD]

If you share the schema I can have a look!

[aplainzetakind]

Here's how it looks in a repl session. If I don't disable the health checks, it doesn't manage to create any nontrivial examples before the checks kick in (usually).

>>> SCHEMA
{'$schema': 'http://json-schema.org/schema#', 'type': 'object', 'properties': {'team': {'type': 'object', 'properties': {'id': {'type': 'string'}, 'slug': {'type': 'string'}, 'name': {'type': 'string'}}, 'required': ['id', 'name', 'slug']}, 'teams': {'type': 'array', 'items': {'type': 'object', 'properties': {'id': {'type': 'string'}, 'slug': {'type': 'string'}, 'name': {'type': 'string'}}, 'required': ['id', 'name', 'slug']}}, 'id': {'type': 'string'}, 'name': {'type': 'string'}, 'slug': {'type': 'string'}, 'isBookmarked': {'type': 'boolean'}, 'isMember': {'type': 'boolean'}, 'hasAccess': {'type': 'boolean'}, 'dateCreated': {'type': 'string'}, 'environments': {'type': 'array', 'items': {'type': 'string'}}, 'eventProcessing': {'type': 'object', 'properties': {'symbolicationDegraded': {'type': 'boolean'}}, 'required': ['symbolicationDegraded']}, 'features': {'type': 'array', 'items': {'type': 'string'}}, 'firstEvent': {'type': ['null', 'string']}, 'firstTransactionEvent': {'type': 'boolean'}, 'hasSessions': {'type': 'boolean'}, 'platform': {'type': ['null', 'string']}, 'platforms': {'type': 'array', 'items': {'type': 'string'}}, 'latestRelease': {'anyOf': [{'type': 'null'}, {'type': 'object', 'properties': {'version': {'type': 'string'}}, 'required': ['version']}]}, 'hasUserReports': {'type': 'boolean'}, 'latestDeploys': {'type': 'null'}}, 'required': ['dateCreated', 'environments', 'eventProcessing', 'features', 'firstEvent', 'firstTransactionEvent', 'hasAccess', 'hasSessions', 'hasUserReports', 'id', 'isBookmarked', 'isMember', 'latestDeploys', 'latestRelease', 'name', 'platform', 'platforms', 'slug', 'team', 'teams']}
>>> @given(from_schema(SCHEMA))
... @settings(max_examples=100, suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow])
... def printit(out):
...     print('\n', out)
... 
>>> printit()

 {'dateCreated': '', 'environments': [], 'eventProcessing': {'symbolicationDegraded': False}, 'features': [], 'firstEvent': None, 'firstTransactionEvent': False, 'hasAccess': False, 'hasSessions': False, 'hasUserReports': False, 'id': '', 'isBookmarked': False, 'isMember': False, 'latestDeploys': None, 'latestRelease': None, 'name': '', 'platform': None, 'platforms': [], 'slug': '', 'team': {'id': '', 'name': '', 'slug': ''}, 'teams': []}

 {'dateCreated': '0', 'environments': [], 'eventProcessing': {'symbolicationDegraded': False}, 'features': [], 'firstEvent': None, 'firstTransactionEvent': False, 'hasAccess': False, 'hasSessions': False, 'hasUserReports': False, 'id': '', 'isBookmarked': False, 'isMember': False, 'latestDeploys': None, 'latestRelease': None, 'name': '', 'platform': None, 'platforms': [], 'slug': '', 'team': {'id': '', 'name': '', 'slug': ''}, 'teams': []}

 {'dateCreated': '', 'environments': [], 'eventProcessing': {'symbolicationDegraded': False}, 'features': ['j', '÷@¿\x92Q\U00092342?\x8b\x18\x03\U000644853[\x8a'], 'firstEvent': '\x14\U00089f50X\x92\x8bR', 'firstTransactionEvent': False, 'hasAccess': True, 'hasSessions': True, 'hasUserReports': False, 'id': 'sÀ\U00094f63', 'isBookmarked': False, 'isMember': True, 'latestDeploys': None, 'latestRelease': None, 'name': 'Øâ\x0et/6\x8b{ßà\U001078d2a', 'platform': '\x00', 'platforms': [''], 'slug': 'ýÛ', 'team': {'id': '', 'name': '\x7ft3a`', 'slug': '\U0006e2e2\x1e\x97ÄØ´', '$\x17ä': {}}, 'teams': []}

 {'dateCreated': '', 'environments': [], 'eventProcessing': {'symbolicationDegraded': True}, 'features': ['', '\x9eèq\U0006da49ÁZHV\U000b3966\x9c', '0\U00060a05', '\x86\U0007b6b9&\x03ec', '\x13\U0001dfbe\x12', '\U000b2365VWÒðá2¹#\U0008d36c\U000fc2a2\x16', '=B\U000dac33', 'z{\x85\x9cï[ô;4ó\x94sP\ue6b0¦', '', '=1\x05', 'BE', '\x8e\x8e¹i'], 'firstEvent': None, 'firstTransactionEvent': False, 'hasAccess': False, 'hasSessions': False, 'hasUserReports': False, 'id': '\U000d850e6\U000c301f\x96Æ^', 'isBookmarked': True, 'isMember': False, 'latestDeploys': None, 'latestRelease': None, 'name': '', 'platform': None, 'platforms': ['¯KE¸$w', 'ñõº', '', 'Pá\U000ff85e'], 'slug': 'x{{1î¥\x80%', 'team': {'id': '', 'name': '\x06Ý', 'slug': '\x92x\U000d7066'}, 'teams': [{'id': '\x80\x90ÂP7û\U00079feeÞ\U000158ff6', 'name': '', 'slug': 'ò'}, {'id': '', 'name': '\U0009e52e\U000cb10dÜ', 'slug': '\U000e08bf', '\U0008a6ed©': [[], True, {}]}], '\x9e': {}}

 {'dateCreated': '0', 'environments': [], 'eventProcessing': {'symbolicationDegraded': False}, 'features': [], 'firstEvent': None, 'firstTransactionEvent': False, 'hasAccess': False, 'hasSessions': False, 'hasUserReports': False, 'id': '', 'isBookmarked': False, 'isMember': False, 'latestDeploys': None, 'latestRelease': None, 'name': '', 'platform': None, 'platforms': [], 'slug': '', 'team': {'id': '', 'name': '', 'slug': ''}, 'teams': []}

 {'dateCreated': '0', 'environments': [], 'eventProcessing': {'symbolicationDegraded': False}, 'features': [], 'firstEvent': None, 'firstTransactionEvent': False, 'hasAccess': False, 'hasSessions': False, 'hasUserReports': False, 'id': '', 'isBookmarked': False, 'isMember': False, 'latestDeploys': None, 'latestRelease': None, 'name': '', 'platform': None, 'platforms': [], 'slug': '', 'team': {'id': '', 'name': '', 'slug': ''}, 'teams': []}

 {'dateCreated': '', 'environments': [''], 'eventProcessing': {'symbolicationDegraded': False}, 'features': [], 'firstEvent': None, 'firstTransactionEvent': False, 'hasAccess': False, 'hasSessions': False, 'hasUserReports': False, 'id': '', 'isBookmarked': False, 'isMember': False, 'latestDeploys': None, 'latestRelease': None, 'name': '', 'platform': None, 'platforms': [], 'slug': '', 'team': {'id': '', 'name': '', 'slug': ''}, 'teams': []}

 {'dateCreated': '0', 'environments': [], 'eventProcessing': {'symbolicationDegraded': False}, 'features': [], 'firstEvent': None, 'firstTransactionEvent': False, 'hasAccess': False, 'hasSessions': False, 'hasUserReports': False, 'id': '', 'isBookmarked': False, 'isMember': False, 'latestDeploys': None, 'latestRelease': None, 'name': '', 'platform': None, 'platforms': [], 'slug': '', 'team': {'id': '', 'name': '', 'slug': ''}, 'teams': []}

 {'dateCreated': '0', 'environments': [], 'eventProcessing': {'symbolicationDegraded': False}, 'features': [], 'firstEvent': None, 'firstTransactionEvent': False, 'hasAccess': False, 'hasSessions': False, 'hasUserReports': False, 'id': '', 'isBookmarked': False, 'isMember': False, 'latestDeploys': None, 'latestRelease': None, 'name': '', 'platform': None, 'platforms': [], 'slug': '', 'team': {'id': '', 'name': '', 'slug': ''}, 'teams': []}

 {'dateCreated': '0', 'environments': [], 'eventProcessing': {'symbolicationDegraded': False}, 'features': [], 'firstEvent': None, 'firstTransactionEvent': False, 'hasAccess': False, 'hasSessions': False, 'hasUserReports': False, 'id': '', 'isBookmarked': False, 'isMember': False, 'latestDeploys': None, 'latestRelease': None, 'name': '', 'platform': None, 'platforms': [], 'slug': '', 'team': {'id': '', 'name': '', 'slug': ''}, 'teams': []}

 {'dateCreated': '0', 'environments': [], 'eventProcessing': {'symbolicationDegraded': False}, 'features': [], 'firstEvent': None, 'firstTransactionEvent': False, 'hasAccess': False, 'hasSessions': False, 'hasUserReports': False, 'id': '', 'isBookmarked': False, 'isMember': False, 'latestDeploys': None, 'latestRelease': None, 'name': '', 'platform': None, 'platforms': [], 'slug': '', 'team': {'id': '', 'name': '', 'slug': ''}, 'teams': []}

 {'dateCreated': '=\U0004cd1dD\U000d0515í', 'environments': ['', 'wi𪍳@\U0005bea8\x08§þé\U000f2ba5', 'ú\U00037998', '¯[§\x11\U000a8700¼j¹\x87', '\x9dÐ_', 'a\U000e403fÆ\x11\x11\U0001eaf27\U000a4a83å𢵶\x08\x1aV', '!', '', '\U00103f4f', '', '\U00037ff4¼\x00', 'ÑëÆ\x12\x16\x9434'], 'eventProcessing': {'symbolicationDegraded': False, '\x84i\U000e4942»ÈB\U0001fa74çÏ\x9a\x82\x8fÎfÚïö\U000cf1a2\x12\U00063211\\': {'': {'b²GB\U00096b18b\x00': {}, 'pß\U0005e202Á': {'\x9dMÿ\x0cHmkì\U0004a1ec': True}}, '³\x1c_\U001014a3': {}}, ':': {}}, 'features': ['mA\U0001e753\xa0\U00091a3c£Øæ\rË\x05þ¾~\U0007b29f', 'V·µ²ÏÇ', '5¦î\x0f', ''], 'firstEvent': None, 'firstTransactionEvent': True, 'hasAccess': True, 'hasSessions': False, 'hasUserReports': False, 'id': '\x12¦Þ\x03\U0007b1fe¿s}\x8d\U0009172d', 'isBookmarked': True, 'isMember': True, 'latestDeploys': None, 'latestRelease': None, 'name': '3\U00030823à\U0001c706', 'platform': None, 'platforms': ['', 'Ìõ\x80I\U0005ad0b\x88\xa00´', '\nzäT', '¾\x90\x83\U000c38e3\x86sÿ\x03', 'Ù\x98Ú:¤2', '', '', '<zq\x97\x88Ä', '', '¦\U00065c40¡\U000e3113', '½\x10', 'Q\x1f\U0003c53f', '', '\x82\U00093f0b\U000c2e5f\x18\U0007378b|ê¹Pu\xa0Ñh$aìu\x90(B"½', '¿', '', '', 'Z', '¬ám\x06àùáâ'], 'slug': '\U000c9743}', 'team': {'id': '\x0eá_h', 'name': '\U00059597$', 'slug': '\x87²á¥\\\x90\U000c4369', '\x9be\x19': {'\x1b': [False, None, 2.220446049250313e-16], '\x8e\x9a': {'3Z¶\x99´I': [], "\x8d'\x9d»": {}, '~': {'': -5.019772086517871e+175}}}, '\x1cF\x18F8(IâÂ\x9f<11E\U000e3836*\x7f.æ\U0002f46b\U00062ce3\x97þwÓ': [], '\U00069180£\U00107434\U001061b9\x0c\x19': {}}, 'teams': []}

[Zac-HD]

OK, two things seem to be causing this:

1. Too much filtering in object name selection; fortunately there's an obvious performance patch! I'll release that shortly, after testing and checking for other low-hanging fruit in the area.
2. Your objects also permit arbitrary unlisted keys, which can be disabled with "additionalProperties": false if that's not intentional for a decent speedup.

(1) mostly fixes the problem; subsequent runs can eventually slow down again because of the way Hypothesis saves diverse (in this case, large and filter-prone) examples in the database. Using the derandomize=True or just database=None settings would be a decent alternative to (2) for that.

[Zac-HD]

Closed in 561a549, version 0.21.0 🚀

[dk-mi]

@Zac-HD great, thank you.