Add paging support for french doctolib + add tests

rbignon · Jul 8, 2021 · 7bd2004 · 7bd2004
1 parent ded2df3
commit 7bd2004
Show file tree

Hide file tree

Showing 2 changed files with 312 additions and 1 deletion.
diff --git a/doctoshotgun.py b/doctoshotgun.py
@@ -96,6 +96,25 @@ def iter_centers_ids(self):
             yield data['searchResultId']
 
     def get_next_page(self):
+        # French doctolib uses data-u attribute of span-element to create the link when user hovers span
+        for span in self.doc.xpath('//div[contains(@class, "next")]/span'):
+            if not span.attrib.has_key('data-u'):
+                continue
+
+            # How to find the corresponding javascript-code:
+            # Press F12 to open dev-tools, select elements-tab, find div.next, right click on element and enable break on substructure change
+            # Hover "Next" element and follow callstack upwards
+            # JavaScript:
+            # var t = (e = r()(e)).data("u")
+            #     , n = atob(t.replace(/\s/g, '').split('').reverse().join(''));
+
+            import base64
+            href = base64.urlsafe_b64decode(''.join(span.attrib['data-u'].split())[::-1]).decode()
+            query = dict(parse.parse_qsl(parse.urlsplit(href).query))
+
+            if 'page' in query:
+                return int(query['page'])
+
         for a in self.doc.xpath('//div[contains(@class, "next")]/a'):
             href = a.attrib['href']
             query = dict(parse.parse_qsl(parse.urlsplit(href).query))

diff --git a/test_browser.py b/test_browser.py
@@ -2,11 +2,12 @@
 from requests.adapters import Response
 import responses
 from html import escape
+import lxml.html as html
 import json
 import datetime
 from woob.browser.browsers import Browser
 from woob.browser.exceptions import ServerError
-from doctoshotgun import DoctolibDE, DoctolibFR, CenterBookingPage
+from doctoshotgun import CentersPage, DoctolibDE, DoctolibFR, CenterBookingPage
 
 # globals
 FIXTURES_FOLDER = "test_fixtures"
@@ -114,6 +115,297 @@ def test_find_centers_de_returns_502_should_fail(tmp_path):
             pass
 
 
+@responses.activate
+def test_get_next_page_fr_should_return_2_on_page_1(tmp_path):
+    """
+    Check that get_next_page returns 2 when we are on page 1 and there is a next page available
+    """
+
+    """
+    Next (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?page=2&ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
+    """
+
+    htmlString = """
+        <div class="next-previous-links">
+            <div class="previous dl-rounded-borders dl-white-bg">
+                <span class="disabled">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
+                    Précédent
+                </span>
+            </div>
+            <div class="next dl-rounded-borders dl-white-bg">
+                <span data-u="=UDMwcTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJnJwcTO20DR1UiQ
+1Uyckl2XlZXa09WbfRXazlmdfZWZyZiM9U2ZhB3PlNmbhJnZvMXZylWY0lmc
+vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
+                    Suivant
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
+                </span>
+            </div>
+        </div>
+        """
+    doc = html.document_fromstring(htmlString)
+
+    response = Response()
+    response._content = b'{}'
+
+    centers_page = CentersPage(browser=Browser(), response=response)
+    centers_page.doc = doc
+    next_page = centers_page.get_next_page()
+    assert next_page == 2
+
+
+@responses.activate
+def test_get_next_page_fr_should_return_3_on_page_2(tmp_path):
+    """
+    Check that get_next_page returns 3 when we are on page 2 and next page is available
+    """
+
+    """ 
+    Previous (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
+    Next (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?page=3&ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
+    """
+
+    htmlString = """
+        <div class="next-previous-links">
+            <div class="previous dl-rounded-borders dl-white-bg">
+                <span data-u="==QNwAzN9QUNlIUNlMHZp9VZ2lGdv12X0l2cpZ3XmVmcmAzN
+5YTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJ3PlNmbhJnZvMXZylWY0lmc
+vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
+                    Précédent
+                </span>
+            </div>
+            <div class="next dl-rounded-borders dl-white-bg">
+                <span data-u="=UDMwcTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJnJwcTO20DR1UiQ
+1Uyckl2XlZXa09WbfRXazlmdfZWZyZyM9U2ZhB3PlNmbhJnZvMXZylWY0lmc
+vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
+                    Suivant
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
+                </span>
+            </div>
+        </div>
+        """
+    doc = html.document_fromstring(htmlString)
+
+    response = Response()
+    response._content = b'{}'
+
+    centers_page = CentersPage(browser=Browser(), response=response)
+    centers_page.doc = doc
+    next_page = centers_page.get_next_page()
+    assert next_page == 3
+
+
+@responses.activate
+def test_get_next_page_fr_should_return_4_on_page_3(tmp_path):
+    """
+    Check that get_next_page returns 4 when we are on page 3 and next page is available
+    """
+
+    """
+    Previous (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?page=2&ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
+    Next (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?page=4&ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005    
+    """
+
+    htmlString = """
+        <div class="next-previous-links">
+            <div class="previous dl-rounded-borders dl-white-bg">
+                <span data-u="=UDMwcTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJnJwcTO20DR1UiQ
+1Uyckl2XlZXa09WbfRXazlmdfZWZyZiM9U2ZhB3PlNmbhJnZvMXZylWY0lmc
+vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
+                    Précédent
+                </span>
+            </div>
+            <div class="next dl-rounded-borders dl-white-bg">
+                <span data-u="=UDMwcTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJnJwcTO20DR1UiQ
+1Uyckl2XlZXa09WbfRXazlmdfZWZyZCN9U2ZhB3PlNmbhJnZvMXZylWY0lmc
+vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
+                    Suivant
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
+                </span>
+            </div>
+        </div>
+        """
+    doc = html.document_fromstring(htmlString)
+
+    response = Response()
+    response._content = b'{}'
+
+    centers_page = CentersPage(browser=Browser(), response=response)
+    centers_page.doc = doc
+    next_page = centers_page.get_next_page()
+    assert next_page == 4
+
+
+def test_get_next_page_fr_should_return_None_on_last_page(tmp_path):
+    """
+    Check that get_next_page returns None when we are on the last page
+    """
+    """
+    Previous (data-u decoded): /vaccination-covid-19-autres-professions-prioritaires/france?page=7&ref_visit_motive_ids%5B%5D=6970&ref_visit_motive_ids%5B%5D=7005
+    """    
+
+    htmlString = """
+        <div class="next-previous-links">
+            <div class="previous dl-rounded-borders dl-white-bg">
+                <span data-u="=UDMwcTPEVTJCVTJzRWafVmdpR3bt9FdpNXa29lZlJnJwcTO20DR1UiQ
+1Uyckl2XlZXa09WbfRXazlmdfZWZyZyN9U2ZhB3PlNmbhJnZvMXZylWY0lmc
+vlmcw1ycu9WazNXZm9mcw1yclJHd1FWL5ETLklmdvNWLu9Wa0FmbpN2YhZ3L">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
+                        Précédent
+                </span>
+            </div>
+            <div class="next dl-rounded-borders dl-white-bg">
+                <span class="disabled">
+                    Suivant
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
+                </span>
+            </div>
+        </div>
+        """
+    doc = html.document_fromstring(htmlString)
+
+    response = Response()
+    response._content = b'{}'
+
+    centers_page = CentersPage(browser=Browser(), response=response)
+    centers_page.doc = doc
+    next_page = centers_page.get_next_page()
+    assert next_page == None
+
+
+@responses.activate
+def test_get_next_page_de_should_return_2_on_page_1(tmp_path):
+    """
+    Check that get_next_page returns 2 when we are on page 1 and next page is available
+    """
+
+    htmlString = """
+        <div class="next-previous-links">
+            <div class="previous dl-rounded-borders dl-white-bg">
+                <span class="disabled">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
+                    vorherige Seite
+                </span>
+            </div>
+            <div class="next dl-rounded-borders dl-white-bg">
+                <a href="/impfung-covid-19-corona/berlin?page=2&amp;ref_visit_motive_ids%5B%5D=6769">
+                    Nächste Seite
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
+                </a>
+            </div>
+        </div>
+        """
+    doc = html.document_fromstring(htmlString)
+
+    response = Response()
+    response._content = b'{}'
+
+    centers_page = CentersPage(browser=Browser(), response=response)
+    centers_page.doc = doc
+    next_page = centers_page.get_next_page()
+    assert next_page == 2
+
+
+@responses.activate
+def test_get_next_page_de_should_return_3_on_page_2(tmp_path):
+    """
+    Check that get_next_page returns 3 when we are on page 2 and next page is available
+    """
+
+    htmlString = """
+        <div class="next-previous-links">
+            <div class="previous dl-rounded-borders dl-white-bg">
+                <a href="/impfung-covid-19-corona/berlin?ref_visit_motive_ids%5B%5D=6769">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
+                    vorherige Seite
+                </a>
+            </div>
+            <div class="next dl-rounded-borders dl-white-bg">
+                <a href="/impfung-covid-19-corona/berlin?page=3&amp;ref_visit_motive_ids%5B%5D=6769">
+                    Nächste Seite
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
+                </a>
+            </div>
+        </div>
+        """
+    doc = html.document_fromstring(htmlString)
+
+    response = Response()
+    response._content = b'{}'
+
+    centers_page = CentersPage(browser=Browser(), response=response)
+    centers_page.doc = doc
+    next_page = centers_page.get_next_page()
+    assert next_page == 3
+
+
+@responses.activate
+def test_get_next_page_de_should_return_4_on_page_3(tmp_path):
+    """
+    Check that get_next_page returns 4 when we are on page 3 and next page is available
+    """
+
+    htmlString = """
+        <div class="next-previous-links">
+            <div class="previous dl-rounded-borders dl-white-bg">
+                <a href="/impfung-covid-19-corona/berlin?page=2&amp;ref_visit_motive_ids%5B%5D=6769">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
+                    vorherige Seite
+                </a>
+            </div>
+            <div class="next dl-rounded-borders dl-white-bg">
+                <a href="/impfung-covid-19-corona/berlin?page=4&amp;ref_visit_motive_ids%5B%5D=6769">
+                    Nächste Seite
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
+                </a>
+            </div>
+        </div>
+        """
+    doc = html.document_fromstring(htmlString)
+
+    response = Response()
+    response._content = b'{}'
+
+    centers_page = CentersPage(browser=Browser(), response=response)
+    centers_page.doc = doc
+    next_page = centers_page.get_next_page()
+    assert next_page == 4
+
+
+def test_get_next_page_de_should_return_None_on_last_page(tmp_path):
+    """
+    Check that get_next_page returns None when we are on the last page
+    """
+
+    htmlString = """
+        <div class="next-previous-links">
+            <div class="previous dl-rounded-borders dl-white-bg">
+                <a href="/impfung-covid-19-corona/berlin?page=5&amp;ref_visit_motive_ids%5B%5D=6769">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.863 7.576l4.859-4.859a.6.6 0 01.848 0l.567.567a.6.6 0 01.001.847L7.288 8l3.85 3.869a.6.6 0 01-.001.847l-.567.567a.6.6 0 01-.848 0L4.863 8.424a.6.6 0 010-.848z"></path></svg>
+                    vorherige Seite
+                </a>
+            </div>
+            <div class="next dl-rounded-borders dl-white-bg">
+                <span class="disabled">
+                    Nächste Seite
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.137 8.424l-4.859 4.859a.6.6 0 01-.848 0l-.567-.567a.6.6 0 010-.847L8.712 8l-3.85-3.869a.6.6 0 010-.847l.567-.567a.6.6 0 01.848 0l4.859 4.859a.6.6 0 010 .848z"></path></svg>
+                </span>
+            </div>
+        </div>
+        """
+    doc = html.document_fromstring(htmlString)
+
+    response = Response()
+    response._content = b'{}'
+
+    centers_page = CentersPage(browser=Browser(), response=response)
+    centers_page.doc = doc
+    next_page = centers_page.get_next_page()
+    assert next_page == None
+
+
 @responses.activate
 def test_book_slots_should_succeed(tmp_path):
     """