From 3286ae317701d2d73fcf44f0324f5893f560f86a Mon Sep 17 00:00:00 2001
From: Xin Dong <104880864+ddl-xin@users.noreply.github.com>
Date: Thu, 31 Aug 2023 15:29:25 -0700
Subject: [PATCH] QE-13022 improve dropdown clicking (#359)

Added steps to handle dynamic dropdown and improved the robustness of
selecting options from a dropdown.
---
 CHANGELOG.md                     |   5 +
 data/www/buttons.html            |   4 +-
 data/www/fuzzy_rules.html        |  18 +--
 pyproject.toml                   |   2 +-
 src/cucu/fuzzy/fuzzy.js          |   6 +-
 src/cucu/steps/dropdown_steps.py | 200 +++++++++++++++++++++++++++++--
 6 files changed, 210 insertions(+), 25 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f65f4442..b58a5474 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project closely adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.158.0
+- Add - steps to handle dynamic dropdown
+- Change - improve the robustness of selecting options in a dropdown
+- Change - fuzzy find won't consider elements with either 0 width or 0 height
+
 ## 0.157.0
 - Add - differentiate between cucu built-in and custom steps
 ## 0.156.0
diff --git a/data/www/buttons.html b/data/www/buttons.html
index b49d87ba..6052cd27 100644
--- a/data/www/buttons.html
+++ b/data/www/buttons.html
@@ -19,8 +19,8 @@
     <input  onclick="update('input type=submit');" type="submit" value="input type=submit"></input>
     <a      onclick="update('a link');" class="button" href="#">a link</a>
     <div    onclick="update('* role=button');" class="button" role="button">* role=button</div>
-    <label for="button-for">button with label for</label><button id="button-for" onclick="update('button with label for');"></button>
-    <label for="button-for">button with label for</label><button id="button-for" onclick="update('button with label for');"></button>
+    <label for="button-for">button with label for</label><button id="button-for" onclick="update('button with label for');">a button</button>
+    <label for="button-for">button with label for</label><button id="button-for" onclick="update('button with label for');">a button</button>
 
     <button onclick="update('disabled button');" disabled>disabled button</button>
     <button onclick="update('aria-disabled button');" aria-disabled="true">aria-disabled button</button>
diff --git a/data/www/fuzzy_rules.html b/data/www/fuzzy_rules.html
index db210cab..9426d6f1 100644
--- a/data/www/fuzzy_rules.html
+++ b/data/www/fuzzy_rules.html
@@ -11,24 +11,24 @@
     <label>value:</label><input id="input" type="text"></input>
     <br/>
     <button onclick="update('button with inner text');">button with inner text</button>
-    <button onclick="update('button with aria-label');" aria-label="button with aria-label"></button>
+    <button onclick="update('button with aria-label');" aria-label="button with aria-label">a button</button>
     <button onclick="update('button with child that has aria-label');"><b aria-label="button with child that has aria-label">child with aria-label</b></button>
-    <button onclick="update('button with title');" title="button with title"></button>
-    <button onclick="update('button with placeholder');" placeholder="button with placeholder"></button>
-    <button onclick="update('button with value');" value="button with value"></button>
+    <button onclick="update('button with title');" title="button with title">a button</button>
+    <button onclick="update('button with placeholder');" placeholder="button with placeholder">a button</button>
+    <button onclick="update('button with value');" value="button with value">a button</button>
     <div>
         <label for="button-with-label">button with for/id label</label>
-        <div><button onclick="update('button with for/id label');" id="button-with-label"></button></div>
+        <div><button onclick="update('button with for/id label');" id="button-with-label">a button</button></div>
     </div>
     <br/>
     <button onclick="update('button with nested label');"><label>button with nested label</label></button>
     <br/>
-    <label>button with immediate previous sibling label</label><button onclick="update('button with immediate previous sibling label');"></button>
+    <label>button with immediate previous sibling label</label><button onclick="update('button with immediate previous sibling label');">a button</button>
     <br/>
-    <label>button with any previous sibling label</label><div></div><div></div><button onclick="update('button with any previous sibling label');"></button>
+    <label>button with any previous sibling label</label><div></div><div></div><button onclick="update('button with any previous sibling label');">a button</button>
     <br/>
     <div>
-      <label>button with previous nested sibling label fake</label><button onclick="update('button with previous nested sibling label fake');"></button>
+      <label>button with previous nested sibling label fake</label><button onclick="update('button with previous nested sibling label fake');">a button</button>
     </div>
     <div>
         <div>
@@ -38,7 +38,7 @@
         </div>
         <div>
             <div>
-                <button onclick="update('button with previous nested sibling label');"></button>
+                <button onclick="update('button with previous nested sibling label');">a button</button>
             </div>
         </div>
     </div>
diff --git a/pyproject.toml b/pyproject.toml
index 7dda10c6..0307c402 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cucu"
-version = "0.157.0"
+version = "0.158.0"
 license = "MIT"
 description = "Easy BDD web testing"
 authors = ["Domino Data Lab <open-source@dominodatalab.com>"]
diff --git a/src/cucu/fuzzy/fuzzy.js b/src/cucu/fuzzy/fuzzy.js
index 005825dc..46af8900 100644
--- a/src/cucu/fuzzy/fuzzy.js
+++ b/src/cucu/fuzzy/fuzzy.js
@@ -20,7 +20,7 @@
                 return (elem.textContent || elem.innerText || jqCucu(elem).text() || '').trim() === match[3].trim();
             },
             vis: function (elem) {
-                return !(jqCucu(elem).is(":hidden") || jqCucu(elem).parents(":hidden").length);
+                return !(jqCucu(elem).is(":hidden") || jqCucu(elem).width() == 0 || jqCucu(elem).height == 0 || jqCucu(elem).parents(":hidden").length);
             }
         }
     );
@@ -87,10 +87,10 @@
                     var attribute_name = attributes[aIndex];
                     if (matcher == 'has_text') {
                         results = jqCucu(thing + '[' + attribute_name + '="' + name + '"]:vis', document.body).toArray();
-                        if (cucu.debug) { console.log('<thing attibute="name"></thing>', results); }
+                        if (cucu.debug) { console.log('<thing attribute="name"></thing>', results); }
                     } else if (matcher == 'contains') {
                         results = jqCucu(thing + '[' + attribute_name + '*="' + name + '"]:vis', document.body).toArray();
-                        if (cucu.debug) { console.log('<thing attibute*="name"></thing>', results); }
+                        if (cucu.debug) { console.log('<thing attribute*="name"></thing>', results); }
                     }
                     elements = elements.concat(results);
                 }
diff --git a/src/cucu/steps/dropdown_steps.py b/src/cucu/steps/dropdown_steps.py
index b5d22f71..aac7d1e9 100644
--- a/src/cucu/steps/dropdown_steps.py
+++ b/src/cucu/steps/dropdown_steps.py
@@ -1,7 +1,20 @@
+import logging
+
 import humanize
+from selenium.common.exceptions import ElementClickInterceptedException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support.ui import Select
+from tenacity import (
+    before_sleep_log,
+    retry_if_result,
+    stop_after_attempt,
+    wait_fixed,
+)
+from tenacity import retry as retrying
 
-from cucu import fuzzy, helpers, retry, step
+from cucu import fuzzy, helpers, logger, retry, step
+from cucu.steps.input_steps import find_input
 
 from . import base_steps
 
@@ -36,10 +49,38 @@ def find_dropdown(ctx, name, index=0):
         index=index,
         direction=fuzzy.Direction.LEFT_TO_RIGHT,
     )
+    if not dropdown:
+        # In case the name is on the top of the dropdown element,
+        # the name is after the ting in DOM. Try the other direction.
+        dropdown = fuzzy.find(
+            ctx.browser,
+            name,
+            [
+                "select",
+                '*[role="combobox"]',
+                '*[role="listbox"]',
+            ],
+            index=index,
+            direction=fuzzy.Direction.RIGHT_TO_LEFT,
+        )
+
+    if dropdown:
+        outer_html = dropdown.get_attribute("outerHTML")
+        logger.debug(f'looked for dropdown "{name}", and found "{outer_html}"')
+    else:
+        logger.debug(f'looked for dropdown "{name}" but found none')
 
     return dropdown
 
 
+@retrying(
+    retry=retry_if_result(lambda result: result is None),
+    stop=stop_after_attempt(10),
+    wait=wait_fixed(0.1),
+    before_sleep=before_sleep_log(logger, logging.DEBUG),
+    reraise=True,
+    retry_error_callback=lambda retry_state: retry_state.outcome.result(),
+)
 def find_dropdown_option(ctx, name, index=0):
     """
     find a dropdown option with the provided name. It only considers
@@ -64,18 +105,51 @@ def find_dropdown_option(ctx, name, index=0):
         [
             "option",
             '*[role="option"]',
+            '*[role="treeitem"]',
+            "*[aria-selected]",
         ],
         index=index,
         direction=fuzzy.Direction.LEFT_TO_RIGHT,
         name_within_thing=True,
     )
+    if option:
+        outer_html = option.get_attribute("outerHTML")
+        logger.debug(
+            f'looked for dropdown option "{name}", and found "{outer_html}"'
+        )
+    else:
+        logger.debug(f'looked for dropdown option "{name}" but found none')
 
-    prefix = "" if index == 0 else f"{humanize.ordinal(index)} "
+    return option
 
-    if option is None:
-        raise RuntimeError(f"unable to find the {prefix}option {name}")
 
-    return option
+def click_dropdown(ctx, dropdown):
+    """
+    Internal method used to simply click a dropdown element
+
+    Args:
+        ctx(object): behave context object used to share data between steps
+        dropdown(WebElement): the dropdown element
+    """
+    ctx.check_browser_initialized()
+
+    if base_steps.is_disabled(dropdown):
+        raise RuntimeError("unable to click the button, as it is disabled")
+
+    logger.debug("clicking dropdown")
+    try:
+        ctx.browser.click(dropdown)
+    except ElementClickInterceptedException:
+        clickable = dropdown
+        while True:
+            # In some cases, the dropdown is blocked by the selected item.
+            # It finds the ancestors of the dropdown that is clickable and click.
+            clickable = clickable.find_element(By.XPATH, "..")
+            try:
+                ctx.browser.click(clickable)
+            except ElementClickInterceptedException:
+                continue
+            break
 
 
 def find_n_select_dropdown_option(ctx, dropdown, option, index=0):
@@ -108,7 +182,7 @@ def find_n_select_dropdown_option(ctx, dropdown, option, index=0):
     else:
         if dropdown_element.get_attribute("aria-expanded") != "true":
             # open the dropdown
-            ctx.browser.click(dropdown_element)
+            click_dropdown(ctx, dropdown_element)
 
         option_element = find_dropdown_option(ctx, option)
 
@@ -117,9 +191,74 @@ def find_n_select_dropdown_option(ctx, dropdown, option, index=0):
                 f'unable to find option "{option}" in dropdown "{dropdown}"'
             )
 
+        logger.debug("clicking dropdown option")
+        ctx.browser.execute("arguments[0].scrollIntoView();", option_element)
         ctx.browser.click(option_element)
 
 
+def find_n_select_dynamic_dropdown_option(ctx, dropdown, option, index=0):
+    """
+    find and select dynamic dropdown option
+
+    parameters:
+      ctx(object): behave context object used to share data between steps
+      name(str):   name that identifies the desired dropdown on screen
+      option(str): name of the option to select
+      index(str):  the index of the dropdown if there are duplicates
+    """
+    ctx.check_browser_initialized()
+
+    dropdown_element = find_dropdown(ctx, dropdown, index)
+
+    if dropdown_element is None:
+        prefix = "" if index == 0 else f"{humanize.ordinal(index)} "
+        raise RuntimeError(f"unable to find the {prefix}dropdown {dropdown}")
+
+    if base_steps.is_disabled(dropdown_element):
+        raise RuntimeError(
+            "unable to select from the dropdown, as it is disabled"
+        )
+
+    if dropdown_element.get_attribute("aria-expanded") != "true":
+        # open the dropdown
+        click_dropdown(ctx, dropdown_element)
+
+    option_element = find_dropdown_option(ctx, option)
+
+    # Use the search feature to make the option visible so cucu can pick it up
+    if option_element is None:
+        dropdown_input = find_input(ctx, dropdown, index)
+        logger.debug(
+            f'option "{option}" is not found, trying to send keys "{option}".'
+        )
+        dropdown_value = dropdown_input.get_attribute("value")
+        if dropdown_value:
+            logger.debug(f"clear dropdown value: {dropdown_value}")
+            dropdown_input.send_keys(
+                Keys.ARROW_RIGHT * len(dropdown_value)
+            )  # make sure the cursor is at the end
+            dropdown_input.send_keys(Keys.BACKSPACE * len(dropdown_value))
+        # After each key stroke there is a request and an update of the option list. To prevent stale element,
+        # we send keys one by one here and try to find the option after each key.
+        for key in option:
+            dropdown_input = find_input(ctx, dropdown, index)
+            logger.debug(f'sending key "{key}"')
+            dropdown_input.send_keys(key)
+            ctx.browser.wait_for_page_to_load()
+            option_element = find_dropdown_option(ctx, option)
+            if option_element:
+                break
+
+    if option_element is None:
+        raise RuntimeError(
+            f'unable to find option "{option}" in dropdown "{dropdown}"'
+        )
+
+    logger.debug("clicking dropdown option")
+    ctx.browser.execute("arguments[0].scrollIntoView();", option_element)
+    ctx.browser.click(option_element)
+
+
 def assert_dropdown_option_selected(
     ctx, dropdown, option, index=0, is_selected=True
 ):
@@ -135,8 +274,10 @@ def assert_dropdown_option_selected(
     ctx.check_browser_initialized()
 
     dropdown_element = find_dropdown(ctx, dropdown, index)
-    selected_option = None
+    if dropdown_element is None:
+        raise RuntimeError(f'unable to find dropdown "{dropdown}"')
 
+    selected_option = None
     if dropdown_element.tag_name == "select":
         select_element = Select(dropdown_element)
         selected_option = select_element.first_selected_option
@@ -161,12 +302,14 @@ def assert_dropdown_option_selected(
     else:
         if dropdown_element.get_attribute("aria-expanded") != "true":
             # open the dropdown to see its options
-            ctx.browser.click(dropdown_element)
+            click_dropdown(ctx, dropdown_element)
 
         selected_option = find_dropdown_option(ctx, option)
 
-        # close the dropdown
-        ctx.browser.click(dropdown_element)
+        if selected_option is None:
+            raise RuntimeError(
+                f'unable to find option "{option}" in dropdown "{dropdown}"'
+            )
 
         if is_selected:
             if selected_option.get_attribute("aria-selected") != "true":
@@ -175,6 +318,9 @@ def assert_dropdown_option_selected(
             if selected_option.get_attribute("aria-selected") == "true":
                 raise RuntimeError(f"{option} is selected")
 
+        # close the dropdown
+        click_dropdown(ctx, dropdown_element)
+
 
 helpers.define_should_see_thing_with_name_steps("dropdown", find_dropdown)
 helpers.define_thing_with_name_in_state_steps(
@@ -186,6 +332,12 @@ def assert_dropdown_option_selected(
 helpers.define_run_steps_if_I_can_see_element_with_name_steps(
     "dropdown", find_dropdown
 )
+helpers.define_action_on_thing_with_name_steps(
+    "dropdown", "click", find_dropdown, click_dropdown, with_nth=True
+)
+helpers.define_thing_with_name_in_state_steps(
+    "dropdown option", "disabled", find_dropdown_option, base_steps.is_disabled
+)
 
 
 @step('I select the option "{option}" from the dropdown "{dropdown}"')
@@ -212,6 +364,34 @@ def wait_to_select_option_from_dropdown(ctx, option, dropdown):
     retry(find_n_select_dropdown_option)(ctx, dropdown, option)
 
 
+@step('I select the option "{option}" from the dynamic dropdown "{dropdown}"')
+def select_option_from_dynamic_dropdown(ctx, option, dropdown):
+    find_n_select_dynamic_dropdown_option(ctx, dropdown, option)
+
+
+@step(
+    'I select the option "{option}" from the "{index:nth}" dynamic dropdown "{dropdown}"'
+)
+def select_option_from_nth_dynamic_dropdown(ctx, option, dropdown, index):
+    find_n_select_dynamic_dropdown_option(ctx, dropdown, option, index)
+
+
+@step(
+    'I wait to select the option "{option}" from the "{index:nth}" dynamic dropdown "{dropdown}"'
+)
+def wait_to_select_option_from_nth_dynamic_dropdown(
+    ctx, option, dropdown, index
+):
+    retry(find_n_select_dynamic_dropdown_option)(ctx, dropdown, option, index)
+
+
+@step(
+    'I wait to select the option "{option}" from the dynamic dropdown "{dropdown}"'
+)
+def wait_to_select_option_from_dynamic_dropdown(ctx, option, dropdown):
+    retry(find_n_select_dynamic_dropdown_option)(ctx, dropdown, option)
+
+
 @step(
     'I should see the option "{option}" is selected on the dropdown "{dropdown}"'
 )