web crawler text splitter for HTML splits too often

[CakeCrusher]

Ex:

from weaviate.classes.query import Filter

collection = client.collections.get(name="web_retrieval")
# response = collection.query.near_text(
#     query="CISE academic advisors are an invaluable resource for students at all stages of their academic careers.",
#     limit=5
#     filters=[]
# )
response = collection.query.fetch_objects(
    filters=Filter.by_property("url").equal("https://www.cise.ufl.edu/academics/advising-information/"),
    limit=10000
)

print(len(response.objects))
for i in response.objects:
    print(i.properties)
    print()

16
{'content': 'Facebook Twitter YouTube LinkedIn Instagram  \nWelcome  \nCurrent Students Faculty & Staff Family & Visitors Alumni & Friends', 'depth': 0.0, 'url': 'https://www.cise.ufl.edu/academics/advising-information/'}

{'content': 'UF News UF Calendar myUFL One.UF Directory', 'depth': 0.0, 'url': 'https://www.cise.ufl.edu/academics/advising-information/'}

{'content': 'Campus', 'depth': 0.0, 'url': 'https://www.cise.ufl.edu/academics/advising-information/'}