Avoiding CDATA nonsense

medialab · Feb 15, 2024 · c2e12a9 · c2e12a9
1 parent 50571ac
commit c2e12a9
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 5 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -17,7 +17,7 @@ charset-normalizer==3.3.2
 dateparser==1.1.6
 ebbe==1.13.2
 json5==0.9.11
-lxml>=4.9.2,<5
+lxml>=4.9.2
 nanoid==2.0.0
 playwright==1.35.0
 playwright_stealth==1.0.5

diff --git a/test/scraper_test.py b/test/scraper_test.py
@@ -97,6 +97,8 @@
     </table>
 """
 
+
+# NOTE: CDATA is handled very differently depending on lxml & bs4 versions
 THE_WORST_HTML = """
     <div>Some text isn't
     it?
@@ -136,8 +138,6 @@
                 <li>Other</li>
                 <li>Again</li>
             </ol>
-            <p>
-            <![CDATA[some very interesting stuff]]></p>
             <p>
                 This is      <span>a large span  </span>
                     with something else over <strong>here</strong>.
@@ -997,8 +997,6 @@ def clean(t):
             Other
             Again
 
-            some very interesting stuff
-
             This is a large span with something else over here.
 
             Hello gorgeous!