From 3c471b8cf1d5aba6d2c2a5ed9a684a7d5bc2761c Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 2 Nov 2018 14:03:26 +0500
Subject: [PATCH 01/21] [tmp] Selector.text and SelectorList.text methods

---
 parsel/selector.py | 25 ++++++++++++++++++++++++-
 setup.py           |  3 ++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/parsel/selector.py b/parsel/selector.py
index f9292a4f..73725bc6 100644
--- a/parsel/selector.py
+++ b/parsel/selector.py
@@ -150,6 +150,15 @@ def attrib(self):
         else:
             return {}
 
+    def text(self, clean_html=True, guess_punct_space=True, guess_layout=True,
+             sep='\n'):
+        return sep.join(
+            x.text(clean_html=clean_html,
+                   guess_punct_space=guess_punct_space,
+                   guess_layout=guess_layout)
+            for x in self
+        )
+
 
 class Selector(object):
     """
@@ -162,7 +171,7 @@ class Selector(object):
     If ``type`` is ``None``, the selector defaults to ``"html"``.
     """
 
-    __slots__ = ['text', 'namespaces', 'type', '_expr', 'root',
+    __slots__ = ['namespaces', 'type', '_expr', 'root',
                  '__weakref__', '_parser', '_csstranslator', '_tostring_method']
 
     _default_type = None
@@ -346,6 +355,20 @@ def attrib(self):
         """
         return dict(self.root.attrib)
 
+    def text(self, clean_html=True, guess_punct_space=True, guess_layout=True):
+        from html_text.html_text import _clean_html, _html_to_text
+        tree = _clean_html(self.root) if clean_html else self.root
+        return _html_to_text(tree,
+                             guess_punct_space=guess_punct_space,
+                             guess_layout=guess_layout)
+
+    # def cleaned(self):
+    #     from html_text.html_text import _clean_html
+    #     root = _clean_html(self.root)
+    #     return self.__class__(root=root, _expr=self._expr,
+    #                           namespaces=self.namespaces,
+    #                           type=self.type)
+
     def __bool__(self):
         """
         Return ``True`` if there is any real content selected or ``False``
diff --git a/setup.py b/setup.py
index 53f6a1c4..575cf40b 100644
--- a/setup.py
+++ b/setup.py
@@ -29,7 +29,8 @@ def has_environment_marker_platform_impl_support():
     'w3lib>=1.19.0',
     'lxml>=2.3',
     'six>=1.5.2',
-    'cssselect>=0.9'
+    'cssselect>=0.9',
+    'html-text>=0.4.1',
 ]
 extras_require = {}
 

From 8dea4cefe9b581f8d3da747126228eb7bce7c3f5 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sat, 17 Nov 2018 15:40:55 +0500
Subject: [PATCH 02/21] [wip] move converting to text to .get method, add
 getall support, .cleaned

---
 parsel/selector.py | 157 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 120 insertions(+), 37 deletions(-)

diff --git a/parsel/selector.py b/parsel/selector.py
index 73725bc6..d3877a04 100644
--- a/parsel/selector.py
+++ b/parsel/selector.py
@@ -6,6 +6,8 @@
 
 import six
 from lxml import etree, html
+from lxml.html.clean import Cleaner
+import html_text
 
 from .utils import flatten, iflatten, extract_regex
 from .csstranslator import HTMLTranslator, GenericTranslator
@@ -121,21 +123,42 @@ def re_first(self, regex, default=None, replace_entities=True):
         else:
             return default
 
-    def getall(self):
+    def getall(self, text=False, cleaner='auto',
+               guess_punct_space=True, guess_layout=True):
         """
         Call the ``.get()`` method for each element is this list and return
         their results flattened, as a list of unicode strings.
-        """
-        return [x.get() for x in self]
+
+        ``text``, ``cleaner``, ``guess_punct_space`` and ``guess_layout``
+        options are passed to :meth:`~.Selector.get`; see
+        :meth:`~.Selector.get` for more details.
+        """
+        return [
+            x.get(
+                text=text,
+                cleaner=cleaner,
+                guess_punct_space=guess_punct_space,
+                guess_layout=guess_layout
+            )
+            for x in self
+        ]
     extract = getall
 
-    def get(self, default=None):
+    def get(self, default=None, text=False, cleaner='auto',
+            guess_punct_space=True, guess_layout=True):
         """
         Return the result of ``.get()`` for the first element in this list.
-        If the list is empty, return the default value.
+        If the list is empty, return the ``default`` value.
+
+        ``text``, ``cleaner``, ``guess_punct_space`` and ``guess_layout``
+        options are passed to :meth:`Selector.get`; see :meth:`~.Selector.get`
+        for more details.
         """
         for x in self:
-            return x.get()
+            return x.get(text=text,
+                         cleaner=cleaner,
+                         guess_punct_space=guess_punct_space,
+                         guess_layout=guess_layout)
         else:
             return default
     extract_first = get
@@ -150,15 +173,6 @@ def attrib(self):
         else:
             return {}
 
-    def text(self, clean_html=True, guess_punct_space=True, guess_layout=True,
-             sep='\n'):
-        return sep.join(
-            x.text(clean_html=clean_html,
-                   guess_punct_space=guess_punct_space,
-                   guess_layout=guess_layout)
-            for x in self
-        )
-
 
 class Selector(object):
     """
@@ -188,6 +202,8 @@ class Selector(object):
     }
     _lxml_smart_strings = False
     selectorlist_cls = SelectorList
+    _text_cleaner = html_text.cleaner
+    _html_cleaner = Cleaner()
 
     def __init__(self, text=None, type=None, namespaces=None, root=None,
                  base_url=None, _expr=None):
@@ -301,30 +317,87 @@ def re_first(self, regex, default=None, replace_entities=True):
         """
         return next(iflatten(self.re(regex, replace_entities=replace_entities)), default)
 
-    def get(self):
+    def get(self, text=False, cleaner='auto',
+            guess_punct_space=True, guess_layout=True):
         """
         Serialize and return the matched nodes in a single unicode string.
         Percent encoded content is unquoted.
-        """
+
+        When ``text`` is False (default), HTML or XML is extracted. Pass
+        ``text=True`` to extract text content (html-text library is used).
+        Text extraction algorithm assumes that the document is an HTML
+        document, and uses HTML-specific rules.
+
+        ``cleaner`` argument allows to clean HTML before extracting the
+        content. Allowed values:
+
+        * "auto" (default) - don't clean when text=False, clean with
+          options tuned for text extraction when text=True;
+        * "text" - clean with options tuned for text extraction: elements
+          like ``<script>`` and ``<style>`` are removed, cleaning options
+          are tuned for speed, assuming text extraction is the end goal;
+        * "html" - use default ``lxml.html.clean.Cleaner``. This is useful
+          if you want to make .get() output more human-readable, but still
+          preserve HTML tags.
+        * None - don't clean, even when ``text=True``. Useful if you have
+          an already cleaned tree, e.g. after calling :meth:`Selector.cleaned`.
+        * custom ``lxml.html.clean.Cleaner`` objects are also supported.
+
+        ``guess_punct_space`` and ``guess_layout`` options allow to customize
+        text extraction algorithm. By default, when ``text=True``,
+        parsel tries to insert newlines and blank lines as appropriate,
+        and be smart about whitespaces around inline tags,
+        so that the text output looks similar to browser's.
+
+        Pass ``guess_punct_space=False`` to disable punctuation handling.
+        This option has no effect when ``text=False``.
+
+        Use ``guess_layout=False`` to avoid adding newlines - content will
+        be just a single line of text, using whitespaces as separators.
+        This option has no effect when ``text=False``.
+        """
+        sel = self
+        if cleaner == 'auto':
+            if text:
+                sel = self.cleaned('text')
+        elif cleaner is not None:
+            sel = self.cleaned(cleaner)
+        tree = sel.root
+
+        if text:
+            return html_text.etree_to_text(tree,
+                guess_punct_space=guess_punct_space,
+                guess_layout=guess_layout
+            )
+
         try:
-            return etree.tostring(self.root,
+            return etree.tostring(tree,
                                   method=self._tostring_method,
                                   encoding='unicode',
                                   with_tail=False)
         except (AttributeError, TypeError):
-            if self.root is True:
+            if tree is True:
                 return u'1'
-            elif self.root is False:
+            elif tree is False:
                 return u'0'
             else:
-                return six.text_type(self.root)
+                return six.text_type(tree)
     extract = get
 
-    def getall(self):
+    def getall(self, text=False, cleaner='auto',
+               guess_punct_space=True, guess_layout=True):
         """
-        Serialize and return the matched node in a 1-element list of unicode strings.
+        Serialize and return the matched node in a 1-element list of unicode
+        strings.
+
+        See :meth:`~.Selector.get` for options.
         """
-        return [self.get()]
+        return [self.get(
+            text=text,
+            cleaner=cleaner,
+            guess_punct_space=guess_punct_space,
+            guess_layout=guess_layout,
+        )]
 
     def register_namespace(self, prefix, uri):
         """
@@ -355,19 +428,29 @@ def attrib(self):
         """
         return dict(self.root.attrib)
 
-    def text(self, clean_html=True, guess_punct_space=True, guess_layout=True):
-        from html_text.html_text import _clean_html, _html_to_text
-        tree = _clean_html(self.root) if clean_html else self.root
-        return _html_to_text(tree,
-                             guess_punct_space=guess_punct_space,
-                             guess_layout=guess_layout)
-
-    # def cleaned(self):
-    #     from html_text.html_text import _clean_html
-    #     root = _clean_html(self.root)
-    #     return self.__class__(root=root, _expr=self._expr,
-    #                           namespaces=self.namespaces,
-    #                           type=self.type)
+    def cleaned(self, cleaner='html'):
+        """
+        Return a copy of a Selector, with underlying subtree cleaned.
+        Allowed values of ``cleaner`` argument:
+
+        * "html" (default) - use default ``lxml.html.clean.Cleaner``;
+        * "text" - clean with options tuned for text extraction: elements
+          like ``<script>`` and ``<style>`` are removed, cleaning options
+          are tuned for speed, assuming text extraction is the end goal;
+        * custom ``lxml.html.clean.Cleaner`` objects are also supported.
+        """
+        if isinstance(cleaner, six.string_types):
+            if cleaner not in {'html', 'text'}:
+                raise ValueError("cleaner must be 'html', 'text' or "
+                                 "an lxml.html.clean.Cleaner instance")
+        if cleaner == 'html':
+            cleaner = self._html_cleaner
+        elif cleaner == 'text':
+            cleaner = self._text_cleaner
+        root = cleaner.clean_html(self.root)
+        return self.__class__(root=root, _expr=self._expr,
+                              namespaces=self.namespaces,
+                              type=self.type)
 
     def __bool__(self):
         """

From da7bb809e0aa0fe98089031e0c212438e2f550ed Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 30 May 2019 19:08:41 +0500
Subject: [PATCH 03/21] bump html-text required version number

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 575cf40b..71c6c84d 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ def has_environment_marker_platform_impl_support():
     'lxml>=2.3',
     'six>=1.5.2',
     'cssselect>=0.9',
-    'html-text>=0.4.1',
+    'html-text>=0.5.1',
 ]
 extras_require = {}
 

From 7bae279df2942200be22ea18fdc95ff10deeb396 Mon Sep 17 00:00:00 2001
From: Shahid Karimi <karimi.shahid@gmail.com>
Date: Fri, 11 Mar 2022 14:48:28 +0500
Subject: [PATCH 04/21] selector text unit tests

---
 tests/test_selector.py | 57 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/tests/test_selector.py b/tests/test_selector.py
index daf0f326..5235e487 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1132,3 +1132,60 @@ def test_set(self) -> None:
             ).extract(),
             ["url", "name", "startDate", "location", "offers"],
         )
+
+
+class SelectorTextTestCase(unittest.TestCase):
+
+    sscls = Selector
+
+    html_body = '''
+            <body>
+                <div class="product">
+                    <div class="name">Product1</div>
+                    <span class="price"><b>Price:</b>100</span>
+                </div>
+                <div class="product">
+                    <div class="name">Product2</div>
+                    <span class="price"><b>Price:</b>200</span>
+                </div>
+            </body>
+            '''
+
+    def test_text_get(self):
+        sel = self.sscls(text=u'<p>title:</h1>some text</h1></p>')
+        txt = sel.get(text=True)
+        self.assertEqual(txt, 'title:some text')
+
+    def test_text_getall(self):
+        sel = self.sscls(text=u'<ul><li>option1</li><li>option2</li></ul>').getall(text=True)
+        self.assertEqual(1, len(sel))
+        self.assertEqual('option1\noption2', sel[0])
+
+    def test_text_cleaned_get(self):
+        sel = self.sscls(text=u'<p>paragraph</p><style>.items</style>').cleaned('html').get(text=True)
+        self.assertEqual('paragraph', sel)
+
+    def test_text_get_guess_punct_space_false(self):
+        sel = self.sscls(text=u'<p>hello<b>"Folks"</b></p>')
+        txt = sel.get(text=True, guess_punct_space=False)
+        self.assertEqual(txt, 'hello "Folks"')
+
+    def test_text_get_guess_layout_false(self):
+        sel = self.sscls(text=u'<ul><li>option1</li><li>option2</li></ul>')
+        txt = sel.get(text=True, guess_layout=False)
+        self.assertEqual(txt, 'option1 option2')
+
+    def test_text_get_guess_layout_true(self):
+        sel = self.sscls(text=u'<ul><li>option1</li><li>option2</li></ul>')
+        txt = sel.get(text=True, guess_layout=True)
+        self.assertEqual(txt, 'option1\noption2')
+
+    def test_text_css_multiple(self):
+        html = self.sscls(text=self.html_body)
+        items = html.css('.product .price').getall(text=True)
+        self.assertEqual(items, [u'Price: 100', u'Price: 200'])
+
+    def test_text_xpath_get(self):
+        html = self.sscls(text=self.html_body)
+        self.assertEqual(1, 1)
+        self.assertEqual(html.xpath('//div[@class="product"]/span').getall(text=True), [u'Price: 100', u'Price: 200'])

From e4733ee99042521d470de0e12ec39d3be74d668b Mon Sep 17 00:00:00 2001
From: Shahid Karimi <karimi.shahid@gmail.com>
Date: Fri, 11 Mar 2022 17:03:36 +0500
Subject: [PATCH 05/21] code formtting

---
 tests/test_selector.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/test_selector.py b/tests/test_selector.py
index 5235e487..6f7cf4aa 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1152,40 +1152,40 @@ class SelectorTextTestCase(unittest.TestCase):
             '''
 
     def test_text_get(self):
-        sel = self.sscls(text=u'<p>title:</h1>some text</h1></p>')
+        sel = self.sscls(text='<p>title:</h1>some text</h1></p>')
         txt = sel.get(text=True)
         self.assertEqual(txt, 'title:some text')
 
     def test_text_getall(self):
-        sel = self.sscls(text=u'<ul><li>option1</li><li>option2</li></ul>').getall(text=True)
+        sel = self.sscls(text='<ul><li>option1</li><li>option2</li></ul>').getall(text=True)
         self.assertEqual(1, len(sel))
         self.assertEqual('option1\noption2', sel[0])
 
     def test_text_cleaned_get(self):
-        sel = self.sscls(text=u'<p>paragraph</p><style>.items</style>').cleaned('html').get(text=True)
+        sel = self.sscls(text='<p>paragraph</p><style>.items</style>').cleaned('html').get(text=True)
         self.assertEqual('paragraph', sel)
 
     def test_text_get_guess_punct_space_false(self):
-        sel = self.sscls(text=u'<p>hello<b>"Folks"</b></p>')
+        sel = self.sscls(text='<p>hello<b>"Folks"</b></p>')
         txt = sel.get(text=True, guess_punct_space=False)
         self.assertEqual(txt, 'hello "Folks"')
 
     def test_text_get_guess_layout_false(self):
-        sel = self.sscls(text=u'<ul><li>option1</li><li>option2</li></ul>')
+        sel = self.sscls(text='<ul><li>option1</li><li>option2</li></ul>')
         txt = sel.get(text=True, guess_layout=False)
         self.assertEqual(txt, 'option1 option2')
 
     def test_text_get_guess_layout_true(self):
-        sel = self.sscls(text=u'<ul><li>option1</li><li>option2</li></ul>')
+        sel = self.sscls(text='<ul><li>option1</li><li>option2</li></ul>')
         txt = sel.get(text=True, guess_layout=True)
         self.assertEqual(txt, 'option1\noption2')
 
     def test_text_css_multiple(self):
         html = self.sscls(text=self.html_body)
         items = html.css('.product .price').getall(text=True)
-        self.assertEqual(items, [u'Price: 100', u'Price: 200'])
+        self.assertEqual(items, ['Price: 100', 'Price: 200'])
 
     def test_text_xpath_get(self):
         html = self.sscls(text=self.html_body)
         self.assertEqual(1, 1)
-        self.assertEqual(html.xpath('//div[@class="product"]/span').getall(text=True), [u'Price: 100', u'Price: 200'])
+        self.assertEqual(html.xpath('//div[@class="product"]/span').getall(text=True), ['Price: 100', 'Price: 200'])

From 857ca72fefb69390f6941b9d5bb53d1d2dc27b8f Mon Sep 17 00:00:00 2001
From: Shahid Karimi <karimi.shahid@gmail.com>
Date: Fri, 11 Mar 2022 18:29:13 +0500
Subject: [PATCH 06/21] code formatting improvements

---
 tests/test_selector.py | 39 ++++++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/tests/test_selector.py b/tests/test_selector.py
index 6f7cf4aa..419041eb 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1138,7 +1138,7 @@ class SelectorTextTestCase(unittest.TestCase):
 
     sscls = Selector
 
-    html_body = '''
+    html_body = """
             <body>
                 <div class="product">
                     <div class="name">Product1</div>
@@ -1149,21 +1149,27 @@ class SelectorTextTestCase(unittest.TestCase):
                     <span class="price"><b>Price:</b>200</span>
                 </div>
             </body>
-            '''
+            """
 
     def test_text_get(self):
-        sel = self.sscls(text='<p>title:</h1>some text</h1></p>')
+        sel = self.sscls(text="<p>title:</h1>some text</h1></p>")
         txt = sel.get(text=True)
-        self.assertEqual(txt, 'title:some text')
+        self.assertEqual(txt, "title:some text")
 
     def test_text_getall(self):
-        sel = self.sscls(text='<ul><li>option1</li><li>option2</li></ul>').getall(text=True)
+        sel = self.sscls(text="<ul><li>option1</li><li>option2</li></ul>").getall(
+            text=True
+        )
         self.assertEqual(1, len(sel))
-        self.assertEqual('option1\noption2', sel[0])
+        self.assertEqual("option1\noption2", sel[0])
 
     def test_text_cleaned_get(self):
-        sel = self.sscls(text='<p>paragraph</p><style>.items</style>').cleaned('html').get(text=True)
-        self.assertEqual('paragraph', sel)
+        sel = (
+            self.sscls(text="<p>paragraph</p><style>.items</style>")
+            .cleaned("html")
+            .get(text=True)
+        )
+        self.assertEqual("paragraph", sel)
 
     def test_text_get_guess_punct_space_false(self):
         sel = self.sscls(text='<p>hello<b>"Folks"</b></p>')
@@ -1171,21 +1177,24 @@ def test_text_get_guess_punct_space_false(self):
         self.assertEqual(txt, 'hello "Folks"')
 
     def test_text_get_guess_layout_false(self):
-        sel = self.sscls(text='<ul><li>option1</li><li>option2</li></ul>')
+        sel = self.sscls(text="<ul><li>option1</li><li>option2</li></ul>")
         txt = sel.get(text=True, guess_layout=False)
-        self.assertEqual(txt, 'option1 option2')
+        self.assertEqual(txt, "option1 option2")
 
     def test_text_get_guess_layout_true(self):
-        sel = self.sscls(text='<ul><li>option1</li><li>option2</li></ul>')
+        sel = self.sscls(text="<ul><li>option1</li><li>option2</li></ul>")
         txt = sel.get(text=True, guess_layout=True)
-        self.assertEqual(txt, 'option1\noption2')
+        self.assertEqual(txt, "option1\noption2")
 
     def test_text_css_multiple(self):
         html = self.sscls(text=self.html_body)
-        items = html.css('.product .price').getall(text=True)
-        self.assertEqual(items, ['Price: 100', 'Price: 200'])
+        items = html.css(".product .price").getall(text=True)
+        self.assertEqual(items, ["Price: 100", "Price: 200"])
 
     def test_text_xpath_get(self):
         html = self.sscls(text=self.html_body)
         self.assertEqual(1, 1)
-        self.assertEqual(html.xpath('//div[@class="product"]/span').getall(text=True), ['Price: 100', 'Price: 200'])
+        self.assertEqual(
+            html.xpath('//div[@class="product"]/span').getall(text=True),
+            ["Price: 100", "Price: 200"],
+        )

From 79410936a9f25f8a56df22704358ba4f078856f0 Mon Sep 17 00:00:00 2001
From: Shahid Karimi <karimi.shahid@gmail.com>
Date: Mon, 4 Apr 2022 23:49:47 +0500
Subject: [PATCH 07/21] removed unwated tests

---
 tests/test_selector.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_selector.py b/tests/test_selector.py
index 419041eb..fc3d985f 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1152,7 +1152,7 @@ class SelectorTextTestCase(unittest.TestCase):
             """
 
     def test_text_get(self):
-        sel = self.sscls(text="<p>title:</h1>some text</h1></p>")
+        sel = self.sscls(text="<p>title:<h1>some text</h1></p>")
         txt = sel.get(text=True)
         self.assertEqual(txt, "title:some text")
 
@@ -1193,7 +1193,6 @@ def test_text_css_multiple(self):
 
     def test_text_xpath_get(self):
         html = self.sscls(text=self.html_body)
-        self.assertEqual(1, 1)
         self.assertEqual(
             html.xpath('//div[@class="product"]/span').getall(text=True),
             ["Price: 100", "Price: 200"],

From d87982d9349b9808dc9c3bcd041e9f6cc0b102c7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 28 Jun 2022 23:46:06 +0500
Subject: [PATCH 08/21] apply black

---
 parsel/selector.py     | 14 +++++++++++---
 tests/test_selector.py |  6 +++---
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/parsel/selector.py b/parsel/selector.py
index 7fe8a8ea..20557376 100644
--- a/parsel/selector.py
+++ b/parsel/selector.py
@@ -514,12 +514,17 @@ def get(
 
         if text:
             return html_text.etree_to_text(
-                tree, guess_punct_space=guess_punct_space, guess_layout=guess_layout
+                tree,
+                guess_punct_space=guess_punct_space,
+                guess_layout=guess_layout,
             )
 
         try:
             return etree.tostring(
-                tree, method=self._tostring_method, encoding="unicode", with_tail=False
+                tree,
+                method=self._tostring_method,
+                encoding="unicode",
+                with_tail=False,
             )
         except (AttributeError, TypeError):
             if tree is True:
@@ -629,7 +634,10 @@ def cleaned(
             cleaner = self._text_cleaner
         root = cleaner.clean_html(self.root)
         return self.__class__(
-            root=root, _expr=self._expr, namespaces=self.namespaces, type=self.type
+            root=root,
+            _expr=self._expr,
+            namespaces=self.namespaces,
+            type=self.type,
         )
 
     def __bool__(self) -> bool:
diff --git a/tests/test_selector.py b/tests/test_selector.py
index a00dacf1..cb62a777 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1266,9 +1266,9 @@ def test_text_get(self):
         self.assertEqual(txt, "title:some text")
 
     def test_text_getall(self):
-        sel = self.sscls(text="<ul><li>option1</li><li>option2</li></ul>").getall(
-            text=True
-        )
+        sel = self.sscls(
+            text="<ul><li>option1</li><li>option2</li></ul>"
+        ).getall(text=True)
         self.assertEqual(1, len(sel))
         self.assertEqual("option1\noption2", sel[0])
 

From 14dadbddfbf91e1727dc6d5332e07d2645d4d0fb Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 28 Jun 2022 23:56:00 +0500
Subject: [PATCH 09/21] fixed failing test

The assertion was wrong
---
 tests/test_selector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_selector.py b/tests/test_selector.py
index cb62a777..d7526804 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1263,7 +1263,7 @@ class SelectorTextTestCase(unittest.TestCase):
     def test_text_get(self):
         sel = self.sscls(text="<p>title:<h1>some text</h1></p>")
         txt = sel.get(text=True)
-        self.assertEqual(txt, "title:some text")
+        self.assertEqual(txt, "title:\n\nsome text")
 
     def test_text_getall(self):
         sel = self.sscls(

From af0d28a45d65c5bbcb00ce61ca2e9ebd0cc3c7af Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 28 Jun 2022 23:57:26 +0500
Subject: [PATCH 10/21] Make new arguments keyword-only

---
 parsel/selector.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/parsel/selector.py b/parsel/selector.py
index 20557376..219a75dc 100644
--- a/parsel/selector.py
+++ b/parsel/selector.py
@@ -181,6 +181,7 @@ def re_first(
 
     def getall(
         self,
+        *,
         text: bool = False,
         cleaner: Union[str, None, Cleaner] = "auto",
         guess_punct_space: bool = True,
@@ -217,6 +218,7 @@ def getall(
     def get(
         self,
         default: Optional[str] = None,
+        *,
         text: bool = False,
         cleaner: Union[str, None, Cleaner] = "auto",
         guess_punct_space: bool = True,
@@ -462,6 +464,7 @@ def re_first(
 
     def get(
         self,
+        *,
         text: bool = False,
         cleaner: Union[str, None, Cleaner] = "auto",
         guess_punct_space: bool = True,
@@ -538,6 +541,7 @@ def get(
 
     def getall(
         self,
+        *,
         text: bool = False,
         cleaner: Union[str, None, Cleaner] = "auto",
         guess_punct_space: bool = True,

From 1737f8365793d0a050102ce3672a1490eeed3d85 Mon Sep 17 00:00:00 2001
From: Shahid Karimi <karimi.shahid@gmail.com>
Date: Fri, 12 Aug 2022 17:26:51 +0500
Subject: [PATCH 11/21] documentation for selector .get() text

---
 docs/usage.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/usage.rst b/docs/usage.rst
index d0a6fb0b..b3a9111e 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -120,6 +120,15 @@ pseudo-elements::
     >>> selector.css('title::text').get()
     'Example website'
 
+Extract text witout ::text
+==========================
+You can extract inner text without specifying ``::text`` in your selctor instead
+an optional paramter text=True in the ``get()`` or ``getall()`` methods. 
+
+    >>> selector.css('title').get(text=True)
+
+You can pass additional paramter ``guess_punct_space``, ``guess_layout`` and ``guess_layout``
+
 As you can see, ``.xpath()`` and ``.css()`` methods return a
 :class:`~parsel.selector.SelectorList` instance, which is a list of new
 selectors. This API can be used for quickly selecting nested data::

From 17ae5e0506da45d1355fb78d614d23c43228c57d Mon Sep 17 00:00:00 2001
From: Shahid Karimi <karimi.shahid@gmail.com>
Date: Fri, 26 Aug 2022 12:19:33 +0500
Subject: [PATCH 12/21] suggested changes in the PR fixed

---
 docs/usage.rst | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/docs/usage.rst b/docs/usage.rst
index b3a9111e..08e4ec5c 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -120,12 +120,10 @@ pseudo-elements::
     >>> selector.css('title::text').get()
     'Example website'
 
-Extract text witout ::text
-==========================
-You can extract inner text without specifying ``::text`` in your selctor instead
+You can extract inner text without specifying ``::text`` in your selector instead
 an optional paramter text=True in the ``get()`` or ``getall()`` methods. 
 
-    >>> selector.css('title').get(text=True)
+    >>> selector.css('#images').get(text=True)
 
 You can pass additional paramter ``guess_punct_space``, ``guess_layout`` and ``guess_layout``
 

From c6580cc466ca0df3f27edc69a0e0feb0996ad9cf Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sun, 13 Nov 2022 17:41:57 +0500
Subject: [PATCH 13/21] Update docs/usage.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/usage.rst | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/docs/usage.rst b/docs/usage.rst
index 08e4ec5c..0b41a5ea 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -120,12 +120,17 @@ pseudo-elements::
     >>> selector.css('title::text').get()
     'Example website'
 
-You can extract inner text without specifying ``::text`` in your selector instead
-an optional paramter text=True in the ``get()`` or ``getall()`` methods. 
+To extract all text of one or more element and all their child elements, 
+formatted as plain text taking into account HTML tags (e.g. ``<br/>`` is 
+translated as a line break), set ``text=True`` in your call to 
+:meth:`~Selector.get` or :meth:`~Selector.getall` instead of including 
+``::text`` (CSS) or ``/text()`` (XPath) in your query: 
 
-    >>> selector.css('#images').get(text=True)
+>>> selector.css('#images').get(text=True)
+'Name: My image 1\nName: My image 2\nName: My image 3\nName: My image 4\nName: My image 5'
 
-You can pass additional paramter ``guess_punct_space``, ``guess_layout`` and ``guess_layout``
+See :meth:`Selector.get` for additional parameters that you can use to change 
+how the extracted plain text is formatted.
 
 As you can see, ``.xpath()`` and ``.css()`` methods return a
 :class:`~parsel.selector.SelectorList` instance, which is a list of new

From ee3e734c8271249ff9117ef4071f7e8af9617a27 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 1 May 2024 18:56:42 +0500
Subject: [PATCH 14/21] fixed typing

---
 parsel/selector.py     | 27 ++++++++-------------------
 tests/test_selector.py | 16 ++++++++--------
 2 files changed, 16 insertions(+), 27 deletions(-)

diff --git a/parsel/selector.py b/parsel/selector.py
index d734f560..e39a55ab 100644
--- a/parsel/selector.py
+++ b/parsel/selector.py
@@ -22,7 +22,7 @@
 )
 from warnings import warn
 
-import html_text  # type: ignore[import]
+import html_text  # type: ignore[import-untyped]
 import jmespath
 from lxml import etree, html
 from lxml.html.clean import Cleaner  # pylint: disable=no-name-in-module
@@ -275,14 +275,6 @@ def getall(
 
     extract = getall
 
-    # TODO: bring types back
-    # @typing.overload
-    # def get(self, default: None = None) -> Optional[str]:
-    #     pass
-    #
-    # @typing.overload
-    # def get(self, default: str) -> str:
-    #     pass
     def get(
         self,
         default: Optional[str] = None,
@@ -291,7 +283,7 @@ def get(
         cleaner: Union[str, None, Cleaner] = "auto",
         guess_punct_space: bool = True,
         guess_layout: bool = True,
-    ) -> Optional[str]:
+    ) -> Any:
         """
         Return the result of ``.get()`` for the first element in this list.
         If the list is empty, return the ``default`` value.
@@ -822,14 +814,11 @@ def get(
             )
 
         try:
-            return typing.cast(
-                str,
-                etree.tostring(
-                    tree,
-                    method=_ctgroup[self.type]["_tostring_method"],
-                    encoding="unicode",
-                    with_tail=False,
-                ),
+            etree.tostring(
+                tree,
+                method=_ctgroup[self.type]["_tostring_method"],
+                encoding="unicode",
+                with_tail=False,
             )
         except (AttributeError, TypeError):
             if tree is True:
@@ -975,7 +964,7 @@ def cleaned(
         else:
             cleaner_obj = cleaner
 
-        root = cleaner_obj.clean_html(self.root)  # type: ignore[type-var]
+        root = cleaner_obj.clean_html(self.root)
         return self.__class__(
             root=root,
             _expr=self._expr,
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 8b5e554f..ae34e9c1 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1356,19 +1356,19 @@ class SelectorTextTestCase(unittest.TestCase):
             </body>
             """
 
-    def test_text_get(self):
+    def test_text_get(self) -> None:
         sel = self.sscls(text="<p>title:<h1>some text</h1></p>")
         txt = sel.get(text=True)
         self.assertEqual(txt, "title:\n\nsome text")
 
-    def test_text_getall(self):
+    def test_text_getall(self) -> None:
         sel = self.sscls(text="<ul><li>option1</li><li>option2</li></ul>").getall(
             text=True
         )
         self.assertEqual(1, len(sel))
         self.assertEqual("option1\noption2", sel[0])
 
-    def test_text_cleaned_get(self):
+    def test_text_cleaned_get(self) -> None:
         sel = (
             self.sscls(text="<p>paragraph</p><style>.items</style>")
             .cleaned("html")
@@ -1376,27 +1376,27 @@ def test_text_cleaned_get(self):
         )
         self.assertEqual("paragraph", sel)
 
-    def test_text_get_guess_punct_space_false(self):
+    def test_text_get_guess_punct_space_false(self) -> None:
         sel = self.sscls(text='<p>hello<b>"Folks"</b></p>')
         txt = sel.get(text=True, guess_punct_space=False)
         self.assertEqual(txt, 'hello "Folks"')
 
-    def test_text_get_guess_layout_false(self):
+    def test_text_get_guess_layout_false(self) -> None:
         sel = self.sscls(text="<ul><li>option1</li><li>option2</li></ul>")
         txt = sel.get(text=True, guess_layout=False)
         self.assertEqual(txt, "option1 option2")
 
-    def test_text_get_guess_layout_true(self):
+    def test_text_get_guess_layout_true(self) -> None:
         sel = self.sscls(text="<ul><li>option1</li><li>option2</li></ul>")
         txt = sel.get(text=True, guess_layout=True)
         self.assertEqual(txt, "option1\noption2")
 
-    def test_text_css_multiple(self):
+    def test_text_css_multiple(self) -> None:
         html = self.sscls(text=self.html_body)
         items = html.css(".product .price").getall(text=True)
         self.assertEqual(items, ["Price: 100", "Price: 200"])
 
-    def test_text_xpath_get(self):
+    def test_text_xpath_get(self) -> None:
         html = self.sscls(text=self.html_body)
         self.assertEqual(
             html.xpath('//div[@class="product"]/span').getall(text=True),

From 69456c16be4299066eb530fd3587537b7c81ca0e Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 1 May 2024 22:05:33 +0500
Subject: [PATCH 15/21] fixed a refactoring issue

---
 parsel/selector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parsel/selector.py b/parsel/selector.py
index e39a55ab..56e76581 100644
--- a/parsel/selector.py
+++ b/parsel/selector.py
@@ -814,7 +814,7 @@ def get(
             )
 
         try:
-            etree.tostring(
+            return etree.tostring(
                 tree,
                 method=_ctgroup[self.type]["_tostring_method"],
                 encoding="unicode",

From a492278ba338cbdf702d3a2f172996829c6a3288 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 8 May 2024 20:35:27 +0500
Subject: [PATCH 16/21] document O(N^2) gotcha

---
 parsel/selector.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/parsel/selector.py b/parsel/selector.py
index 56e76581..aa052269 100644
--- a/parsel/selector.py
+++ b/parsel/selector.py
@@ -262,6 +262,16 @@ def getall(
         ``text``, ``cleaner``, ``guess_punct_space`` and ``guess_layout``
         options are passed to :meth:`~.Selector.get`; see
         :meth:`~.Selector.get` for more details.
+
+        .. note::
+
+            When either text extraction or cleaning is requested, they're
+            performed on each element in the list individually. So, if you match
+            nested elements (i.e. both parent and descendant), cleaning or
+            text extraction could be run multiple times on the same part
+            of the tree. For example, ``selector.xpath("*").getall(text=True)``
+            has O(N^2) complexity regarding the number of nodes in the tree,
+            not O(N).
         """
         return [
             x.get(

From 8b4ae25a43bcc470b312e520b865887de6424228 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 8 May 2024 21:38:19 +0500
Subject: [PATCH 17/21] make flake8 config compatible with black

---
 .flake8                 | 3 ++-
 .pre-commit-config.yaml | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.flake8 b/.flake8
index 7e5efc63..cffc2c65 100644
--- a/.flake8
+++ b/.flake8
@@ -1,5 +1,6 @@
 [flake8]
-ignore = E203,W503
+max-line-length = 88
+ignore = E203,W503,E701
 per-file-ignores =
     docs/conftest.py:E501
     parsel/csstranslator.py:E501
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index db43480a..42a15fc5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,14 +4,14 @@ repos:
   hooks:
   - id: bandit
     args: [-r, -c, .bandit.yml]
-- repo: https://github.com/PyCQA/flake8
-  rev: 7.0.0
-  hooks:
-  - id: flake8
 - repo: https://github.com/psf/black.git
   rev: 24.2.0
   hooks:
   - id: black
+- repo: https://github.com/PyCQA/flake8
+  rev: 7.0.0
+  hooks:
+  - id: flake8
 - repo: https://github.com/pycqa/isort
   rev: 5.13.2
   hooks:

From ccaaa5bab98cc588e1fb11198e93f6ca8b708173 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 8 May 2024 21:39:02 +0500
Subject: [PATCH 18/21] refactor text and cleaning tests; add more of them

---
 tests/test_selector.py          |  65 -------------------
 tests/test_text_and_cleaning.py | 109 ++++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+), 65 deletions(-)
 create mode 100644 tests/test_text_and_cleaning.py

diff --git a/tests/test_selector.py b/tests/test_selector.py
index ae34e9c1..30806607 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1339,71 +1339,6 @@ def test_set(self) -> None:
         )
 
 
-class SelectorTextTestCase(unittest.TestCase):
-
-    sscls = Selector
-
-    html_body = """
-            <body>
-                <div class="product">
-                    <div class="name">Product1</div>
-                    <span class="price"><b>Price:</b>100</span>
-                </div>
-                <div class="product">
-                    <div class="name">Product2</div>
-                    <span class="price"><b>Price:</b>200</span>
-                </div>
-            </body>
-            """
-
-    def test_text_get(self) -> None:
-        sel = self.sscls(text="<p>title:<h1>some text</h1></p>")
-        txt = sel.get(text=True)
-        self.assertEqual(txt, "title:\n\nsome text")
-
-    def test_text_getall(self) -> None:
-        sel = self.sscls(text="<ul><li>option1</li><li>option2</li></ul>").getall(
-            text=True
-        )
-        self.assertEqual(1, len(sel))
-        self.assertEqual("option1\noption2", sel[0])
-
-    def test_text_cleaned_get(self) -> None:
-        sel = (
-            self.sscls(text="<p>paragraph</p><style>.items</style>")
-            .cleaned("html")
-            .get(text=True)
-        )
-        self.assertEqual("paragraph", sel)
-
-    def test_text_get_guess_punct_space_false(self) -> None:
-        sel = self.sscls(text='<p>hello<b>"Folks"</b></p>')
-        txt = sel.get(text=True, guess_punct_space=False)
-        self.assertEqual(txt, 'hello "Folks"')
-
-    def test_text_get_guess_layout_false(self) -> None:
-        sel = self.sscls(text="<ul><li>option1</li><li>option2</li></ul>")
-        txt = sel.get(text=True, guess_layout=False)
-        self.assertEqual(txt, "option1 option2")
-
-    def test_text_get_guess_layout_true(self) -> None:
-        sel = self.sscls(text="<ul><li>option1</li><li>option2</li></ul>")
-        txt = sel.get(text=True, guess_layout=True)
-        self.assertEqual(txt, "option1\noption2")
-
-    def test_text_css_multiple(self) -> None:
-        html = self.sscls(text=self.html_body)
-        items = html.css(".product .price").getall(text=True)
-        self.assertEqual(items, ["Price: 100", "Price: 200"])
-
-    def test_text_xpath_get(self) -> None:
-        html = self.sscls(text=self.html_body)
-        self.assertEqual(
-            html.xpath('//div[@class="product"]/span').getall(text=True),
-            ["Price: 100", "Price: 200"],
-        )
-
-
 class SelectorBytesInput(Selector):
     def __init__(
         self,
diff --git a/tests/test_text_and_cleaning.py b/tests/test_text_and_cleaning.py
new file mode 100644
index 00000000..8b664cd9
--- /dev/null
+++ b/tests/test_text_and_cleaning.py
@@ -0,0 +1,109 @@
+from lxml.html.clean import Cleaner
+
+from parsel import Selector
+
+HTML_BODY = """
+<body>
+    <div class="product">
+        <div class="name">Product1</div>
+        <span class="price"><b>Price:</b>100</span>
+    </div>
+    <div class="product">
+        <div class="name">Product2</div>
+        <span class="price"><b>Price:</b>200</span>
+    </div>
+</body>
+"""
+
+
+def test_text_get() -> None:
+    sel = Selector("<p>title:<h1>some text</h1></p>")
+    txt = sel.get(text=True)
+    assert txt == "title:\n\nsome text"
+
+
+def test_text_getall() -> None:
+    sel = Selector("<ul><li>option1</li><li>option2</li></ul>")
+
+    assert sel.getall(text=True) == ["option1\noption2"]
+    assert sel.css("li").getall(text=True) == ["option1", "option2"]
+
+
+def test_cleaned() -> None:
+    div_html = "<div><script>SCRIPT</script>" "<style>STYLE</style><p>hello</p><div>"
+    sel = Selector(div_html)
+    assert sel.css("script").getall() == ["<script>SCRIPT</script>"]
+    assert sel.cleaned().css("script").getall() == []
+
+    assert len(sel.css("script")) == 1
+    assert len(sel.css("style")) == 1
+    assert len(sel.css("p")) == 1
+
+    assert len(sel.cleaned().css("script")) == 0
+    assert len(sel.cleaned().css("style")) == 1
+    assert len(sel.cleaned().css("p")) == 1
+
+
+def test_cleaned_options() -> None:
+    div_html = "<div><script>SCRIPT</script>" "<style>STYLE</style><p>hello</p><div>"
+    sel = Selector(div_html)
+    assert len(sel.css("script")) == 1
+    assert len(sel.css("style")) == 1
+    assert len(sel.css("p")) == 1
+
+    assert len(sel.cleaned().css("script")) == 0
+    assert len(sel.cleaned().css("style")) == 1
+    assert len(sel.cleaned().css("p")) == 1
+
+    assert len(sel.cleaned("html").css("script")) == 0
+    assert len(sel.cleaned("html").css("style")) == 1
+    assert len(sel.cleaned("html").css("p")) == 1
+
+    assert len(sel.cleaned("text").css("script")) == 0
+    assert len(sel.cleaned("text").css("style")) == 0
+    assert len(sel.cleaned("text").css("p")) == 1
+
+    cleaner = Cleaner(kill_tags=["p"], scripts=False, style=False)
+    assert len(sel.cleaned(cleaner).css("script")) == 1
+    assert len(sel.cleaned(cleaner).css("style")) == 1
+    assert len(sel.cleaned(cleaner).css("p")) == 0
+
+
+def test_get_cleaner() -> None:
+    div_html = "<div><script>SCRIPT</script><style>STYLE</style><p>P</p></div>"
+    sel = Selector(div_html)
+    cleaner = Cleaner(kill_tags=["p"], scripts=False, style=False)
+
+    assert sel.get(text=True) == "P"
+    assert sel.get(text=True, cleaner=None) == "SCRIPT STYLE\n\nP"
+    assert sel.get(text=True, cleaner="html") == "STYLE\n\nP"
+    assert sel.get(text=True, cleaner="text") == "P"
+    assert sel.get(text=True, cleaner=cleaner) == "SCRIPT STYLE"
+
+    div = sel.css("div")
+    assert div.get() == div_html
+    assert div.get(cleaner=None) == div_html
+    assert div.get(cleaner="html") == "<div><style>STYLE</style><p>P</p></div>"
+    assert div.get(cleaner="text") == "<div><p>P</p></div>"
+    assert (
+        div.get(cleaner=cleaner)
+        == "<div><script>SCRIPT</script><style>STYLE</style></div>"
+    )
+
+
+def test_guess_punct_space() -> None:
+    sel = Selector('<p>hello<b>"Folks"</b></p>')
+    assert sel.get(text=True, guess_punct_space=False) == 'hello "Folks"'
+    assert sel.get(text=True, guess_punct_space=True) == 'hello"Folks"'
+
+    assert sel.getall(text=True, guess_punct_space=False) == ['hello "Folks"']
+    assert sel.getall(text=True, guess_punct_space=True) == ['hello"Folks"']
+
+
+def test_guess_layout() -> None:
+    sel = Selector("<ul><li>option1</li><li>option2</li></ul>")
+    assert sel.get(text=True, guess_layout=False) == "option1 option2"
+    assert sel.get(text=True, guess_layout=True) == "option1\noption2"
+
+    assert sel.getall(text=True, guess_layout=False) == ["option1 option2"]
+    assert sel.getall(text=True, guess_layout=True) == ["option1\noption2"]

From 4eea4fa0c5cdacf58820d115b77eb2dc3f7741f7 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 8 May 2024 21:39:35 +0500
Subject: [PATCH 19/21] fixed default .cleaned cleaner value

---
 parsel/selector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parsel/selector.py b/parsel/selector.py
index aa052269..104db9d4 100644
--- a/parsel/selector.py
+++ b/parsel/selector.py
@@ -949,7 +949,7 @@ def attrib(self) -> Dict[str, str]:
         return dict(self.root.attrib)
 
     def cleaned(
-        self: _SelectorType, cleaner: Union[str, Cleaner] = "auto"
+        self: _SelectorType, cleaner: Union[str, Cleaner] = "html"
     ) -> _SelectorType:
         """
         Return a copy of a Selector, with underlying subtree cleaned.

From 27c9919a67eeef11231e43a93f27cf5c60c8516d Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 8 May 2024 22:47:54 +0500
Subject: [PATCH 20/21] fixed black formatting went wrong

---
 tests/test_text_and_cleaning.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_text_and_cleaning.py b/tests/test_text_and_cleaning.py
index 8b664cd9..a4b6ea75 100644
--- a/tests/test_text_and_cleaning.py
+++ b/tests/test_text_and_cleaning.py
@@ -30,7 +30,7 @@ def test_text_getall() -> None:
 
 
 def test_cleaned() -> None:
-    div_html = "<div><script>SCRIPT</script>" "<style>STYLE</style><p>hello</p><div>"
+    div_html = "<div><script>SCRIPT</script><style>STYLE</style><p>hello</p><div>"
     sel = Selector(div_html)
     assert sel.css("script").getall() == ["<script>SCRIPT</script>"]
     assert sel.cleaned().css("script").getall() == []
@@ -45,7 +45,7 @@ def test_cleaned() -> None:
 
 
 def test_cleaned_options() -> None:
-    div_html = "<div><script>SCRIPT</script>" "<style>STYLE</style><p>hello</p><div>"
+    div_html = "<div><script>SCRIPT</script><style>STYLE</style><p>hello</p><div>"
     sel = Selector(div_html)
     assert len(sel.css("script")) == 1
     assert len(sel.css("style")) == 1

From 852bbef86308515b0b873b3eaca4e26523dff6ee Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 8 May 2024 22:48:13 +0500
Subject: [PATCH 21/21] fix docs references

---
 docs/usage.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/usage.rst b/docs/usage.rst
index 7cfa2fce..0b97d8b2 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -143,13 +143,14 @@ pseudo-elements::
 To extract all text of one or more element and all their child elements, 
 formatted as plain text taking into account HTML tags (e.g. ``<br/>`` is 
 translated as a line break), set ``text=True`` in your call to 
-:meth:`~Selector.get` or :meth:`~Selector.getall` instead of including 
+:meth:`~parsel.selector.Selector.get` or
+:meth:`~parsel.selector.Selector.getall` instead of including
 ``::text`` (CSS) or ``/text()`` (XPath) in your query::
 
     >>> selector.css('#images').get(text=True)
     'Name: My image 1\nName: My image 2\nName: My image 3\nName: My image 4\nName: My image 5'
 
-See :meth:`Selector.get` for additional parameters that you can use to change 
+See :meth:`Selector.get` for additional parameters that you can use to change
 how the extracted plain text is formatted.
 
 As you can see, ``.xpath()`` and ``.css()`` methods return a