Seleniumbase UC + Proxies - Can't bypass Cloudflare Captcha and Detected location - on Pixelscan

[misterpeople]

I am a new developer
I'm using SeleniumBase UC and have added dedicated residential proxies to my setup. Despite this, I'm faceing some issue.

After I login to the account with seleniumbase UC, when I get to a certain page in that website I face Cloudflare "Verifying you are human" and then I am unable to continue since its not working to click on "verify" button manually (by manually I mean that I am clicking manually with my mouse because I haven't automated it yet since its not working.
Here's a brief overview of my setup:

I have added Proxies and I spoof the location to the proxy location.
Below is the some of the code I am using with demo data:

import json
import random
import logging
import telebot
from PIL import Image
from io import BytesIO
import time
import requests
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from seleniumbase import SB
from fake_useragent import UserAgent
import base64
from PIL import ImageEnhance
import numpy as np

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# Proxy settings
PROXY_HOST = "proxy.example.com"
PROXY_PORT = "12345"
PROXY_USER = "demo_user"
PROXY_PASS = "demo_pass"

# Construct the proxy URL
proxy_url = f"{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}"

# User credentials to login
username = "demo_username"
password = "demo_password"

# Custom headers
custom_headers = {
    "User-Agent": "your-user-agent",
    "Accept-Language": "en-US,en;q=0.9",
    "Custom-Header": "your-custom-value"
}

def generate_user_agent():
    ua = UserAgent()
    user_agent = ua.random
    return user_agent

def set_custom_headers(sb, headers):
    for key, value in headers.items():
        sb.driver.execute_cdp_cmd(
            'Network.setExtraHTTPHeaders',
            {'headers': {key: value}}
        )

def solve_captcha(sb):
    try:
        logging.info("Starting CAPTCHA solving")
        captcha_element = WebDriverWait(sb.driver, 30).until(
            EC.presence_of_element_located((By.XPATH, '//*[@id="captchaImage"]'))
        )
        captcha_src = captcha_element.get_attribute('src')

        if captcha_src and 'blob:' in captcha_src:
            logging.info("CAPTCHA loaded successfully")

            screenshot = sb.driver.get_screenshot_as_png()
            screenshot = Image.open(BytesIO(screenshot))

            captcha_location = captcha_element.location
            captcha_size = captcha_element.size
            left = captcha_location['x']
            top = captcha_location['y']
            right = left + captcha_size['width']
            bottom = top + captcha_size['height']

            captcha_image = screenshot.crop((left, top, right, bottom))

            # Resize the CAPTCHA image
            new_size = (1000, 278)
            resized_captcha_image = captcha_image.resize(new_size, Image.LANCZOS)

            # Convert image to grayscale
            gray_image = resized_captcha_image.convert('L')

            # Enhance the image
            enhancer = ImageEnhance.Contrast(gray_image)
            enhanced_image = enhancer.enhance(2.0)

            # Apply a threshold to get a binary image
            binary_image = enhanced_image.point(lambda p: p > 128 and 255)

            # Convert to numpy array
            binary_array = np.array(binary_image)

            # Remove the line
            line_height = binary_array.shape[0] // 2
            line_thickness = 3
            binary_array[line_height-line_thickness:line_height+line_thickness, :] = 255

            # Convert back to image
            processed_image = Image.fromarray(binary_array)

            # Save the processed image
            processed_image_path = "demo_captcha.png"
            processed_image.save(processed_image_path)

            # Send the processed image file path to the Telegram chat
            bot.send_message(chat_id, f"Processed CAPTCHA image saved at: {processed_image_path}")

            with open(processed_image_path, 'rb') as img_file:
                bot.send_photo(chat_id, img_file, caption="Please enter the CAPTCHA text.")

            bot.send_message(chat_id, "Please enter the CAPTCHA text.")

            global captcha_text
            captcha_text = None

            while captcha_text is None:
                time.sleep(1)

            captcha_response_element = WebDriverWait(sb.driver, 30).until(
                EC.presence_of_element_located((By.ID, 'extension_atlasCaptchaResponse'))
            )
            captcha_response_element.clear()
            captcha_response_element.send_keys(captcha_text)
            logging.info(f"Entered CAPTCHA: {captcha_text}")

            sb.driver.find_element(By.ID, 'continue').click()

            WebDriverWait(sb.driver, 20).until(
                EC.invisibility_of_element((By.ID, 'simplemodal-overlay')))
            time.sleep(5)

            if "Captcha Validation is not Successful" not in sb.driver.page_source:
                logging.info("CAPTCHA solved")
                return True
            else:
                logging.info("CAPTCHA validation failed, retrying...")
                bot.send_message(chat_id, "CAPTCHA validation failed, retrying...")
                sb.driver.find_element(By.ID, 'captchaRefreshImage').click()
                time.sleep(2)
                return solve_captcha(sb)
        else:
            logging.info("CAPTCHA not fully loaded or src is invalid, retrying...")
            time.sleep(5)
            return solve_captcha(sb)
    except Exception as e:
        logging.error(f"Error solving CAPTCHA: {e}")
        bot.send_message(chat_id, f"Error solving CAPTCHA: {e}")
        return False

def main():
    sb_instance = SB(uc=True, pls="none", proxy=proxy_url, incognito=True)
    try:
        with sb_instance as sb:
            logging.info("Starting browser session")

            # Set the user agent for Windows
            user_agent = generate_user_agent()
            sb.driver.execute_cdp_cmd(
                "Network.setUserAgentOverride",
                {"userAgent": user_agent}
            )
            logging.info(f"User agent set to: {user_agent}")
            bot.send_message(chat_id, f"User Agent: {user_agent}")

            # Set custom headers
            set_custom_headers(sb, custom_headers)
            logging.info("Custom headers set")

            sb.open("https://www.demo-website.com/en-US/")
            bot.send_message(chat_id, "Opened demo website page")
            logging.info("Opened demo website page")

            tz_params = {'timezoneId': 'Asia/Kolkata'}
            sb.driver.execute_cdp_cmd('Emulation.setTimezoneOverride', tz_params)
            bot.send_message(chat_id, "Set timezone to Asia/Kolkata")
            logging.info("Set timezone to Asia/Kolkata")

            sb.wait_for_element_visible("body", timeout=60)
            bot.send_message(chat_id, "Page loaded completely")
            logging.info("Page loaded completely")

            WebDriverWait(sb.driver, 60).until(EC.presence_of_element_located((By.ID, "signInName")))
            WebDriverWait(sb.driver, 60).until(EC.presence_of_element_located((By.ID, "password")))
            bot.send_message(chat_id, "Login elements found")
            logging.info("Login elements found")

            sb.type("#signInName", username)
            bot.send_message(chat_id, "Username added")
            logging.info("Username added")
            sb.type("#password", password)
            bot.send_message(chat_id, "Password added")
            logging.info("Password added")

            if solve_captcha(sb):
                bot.send_message(chat_id, "CAPTCHA solved successfully")
                logging.info("CAPTCHA solved successfully")
                sb.click('button[type="submit"]')
                bot.send_message(chat_id, "Clicked submit button")
                logging.info("Clicked submit button")
            else:
                bot.send_message(chat_id, "Failed to solve CAPTCHA")
                logging.info("Failed to solve CAPTCHA")

    except Exception as e:
        logging.error(f"Error during login process: {e}")
        bot.send_message(chat_id, f"Error during login process: {e}")
        sb_instance.driver.quit()

[mdmintz]

The easiest way to figure out why you're being detected is to hit https://pixelscan.net/.

The example for that bypasses bot-detection on there successfully: SeleniumBase/examples/raw_pixelscan.py

The example scripts that bypass Cloudflare are working successfully:

• SeleniumBase/examples/raw_uc_mode.py
• SeleniumBase/examples/raw_bing_captcha.py
• SeleniumBase/examples/raw_ahrefs.py
• SeleniumBase/examples/raw_games.py

I saw lots of things from your script that could get you detected:

• Changing the User-Agent (SeleniumBase automatically gives you one that works)
• Changing the Page Load Strategy, pls. (The default one is best)
• Using WebDriverWait (SeleniumBase has its own built-in waits)
• Using click() when you should be using driver.uc_click()

You should review the 2nd UC Mode Video tutorial:

https://www.youtube.com/watch?v=2pTpBtaE7SQ

[misterpeople]

I tried to do the changes you suggested but I am still getting detected so I went to a more simple approach and tried pixelscan, it worked with this script:

# Proxy settings (Demo)
PROXY_HOST = "demo.proxy.com"
PROXY_PORT = "12345"
PROXY_USER = "demo_user"
PROXY_PASS = "demo_pass"

# Construct the proxy URL
proxy_url = f"{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}"

def set_timezone(sb):
    tz_params = {'timezoneId': 'Asia/Kolkata'}
    sb.driver.execute_cdp_cmd('Emulation.setTimezoneOverride', tz_params)
    logging.info("Set timezone to Asia/Kolkata")

with SB(uc=True, incognito=True, test=True, proxy=proxy_url) as sb:
    sb.driver.uc_open_with_reconnect("https://pixelscan.net/", 10)
    sb.remove_elements("jdiv")  # Remove chat widgets
    set_timezone(sb)
    sb.refresh()  # Refresh the page after setting the timezone

    sb.assert_text("No automation framework detected", "pxlscn-bot-detection")
    not_masking = "You are not masking your fingerprint"
    sb.assert_text(not_masking, "pxlscn-fingerprint-masking")
    sb.highlight("span.text-success", loops=8)
    sb.sleep(1)
    sb.highlight("pxlscn-fingerprint-masking div", loops=9, scroll=False)
    sb.sleep(1)
    sb.highlight("div.bot-detection-context", loops=10, scroll=False)
    sb.sleep(2)`

At first it did not work when I applied the spoofed timezone before opening the page for some weird reason no idea why, but then I applied the spoofed time zone after opening the webpage and refreshed the page and It finally worked and pixelscan did not detect anything suspicious, but then I tried the SAME approach on gitlab.com at it did not work:

import os
from seleniumbase import SB
import logging

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# Proxy settings (Demo)
PROXY_HOST = "demo.proxy.com"
PROXY_PORT = "12345"
PROXY_USER = "demo_user"
PROXY_PASS = "demo_pass"

# Construct the proxy URL
proxy_url = f"{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}"

def set_timezone(sb):
    tz_params = {'timezoneId': 'Asia/Kolkata'}
    sb.driver.execute_cdp_cmd('Emulation.setTimezoneOverride', tz_params)
    logging.info("Set timezone to Asia/Kolkata")

with SB(uc=True, incognito=True, test=True, proxy=proxy_url) as sb:
    url = "https://gitlab.com/users/sign_in"
    sb.driver.uc_open_with_reconnect(url, 3)
    if not sb.is_text_visible("Username", '[for="user_login"]'):
        sb.driver.uc_open_with_reconnect(url, 4)
    sb.assert_text("Username", '[for="user_login"]', timeout=99)
    sb.assert_element('label[for="user_login"]')
    sb.highlight('button:contains("Sign in")')
    sb.highlight('h1:contains("GitLab.com")')
    sb.post_message("SeleniumBase wasn't detected", duration=99)

    # Set timezone
    set_timezone(sb)
    sb.refresh()  # Refresh the page after setting the timezone

    # Verify the page after refreshing
    sb.assert_text("Username", '[for="user_login"]', timeout=99)
    sb.assert_element('label[for="user_login"]')
    sb.highlight('button:contains("Sign in")')
    sb.highlight('h1:contains("GitLab.com")')
    sb.post_message("SeleniumBase wasn't detected", duration=99)

It's getting stuck on the CAPTCHA page and won't pass it. I can't figure out a way to bypass it. I spoofed the timezone to match the proxy, which works on Pixelscan, but not on Gitlab.com, which has the same type of CAPTCHA as the actual website I need to access. Any suggestions?

[mdmintz]

Each time chromedriver disconnects from Chrome to avoid detection (after the uc_open_* methods and uc_click()), the settings changed by driver.execute_cdp_cmd() are reset. In your case, since you want to make sure the timezone stays changed, use a different way of setting the timezone, such as https://www.browserstack.com/guide/change-time-zone-in-chrome-for-testing or via a Chrome extension. SeleniumBase lets you load a Chrome extension directory via extension_dir=DIR. So you'll have to find one of those extensions, extract it, modify it to change the default timezone, and then tell SeleniumBase to use it when spinning up a browser in UC Mode. Also note that extensions won't work in Incognito Mode or Guest Mode (https://support.google.com/chrome_webstore/answer/2664769?hl=en), so plan accordingly.

[mdmintz]

There might be a way with the selenium-stealth integration: #2404
UC Mode makes the browser appear human-controlled, but all the fingerprint-masking is done by external integrations and extensions.

[misterpeople]

I tried, it did not work, any idea what can be done or if im doing something wrong? im getting a feeling like its NOT going to happen :(

# Proxy settings
PROXY_HOST = "demo"
PROXY_PORT = "demo"
PROXY_USER = "demo"
PROXY_PASS = "demo"

# Construct the proxy URL
proxy_url = f"{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}"

# Path to the extracted extension directory
extension_dir = r"C:\Users\PC\Desktop\Test\2"

# Function to set up stealth mode
def set_up_stealth(driver):
    stealth(driver,
            languages=["en-US", "en"],
            vendor="Google Inc.",
            platform="Win32",
            webgl_vendor="Intel Inc.",
            renderer="Intel Iris OpenGL Engine",
            fix_hairline=True)

# Main script execution
with SB(uc=True, test=True, proxy=proxy_url, extension_dir=extension_dir) as sb:
    url = "https://gitlab.com/users/sign_in"
    sb.driver.uc_open_with_reconnect(url, 3)

    # Set up stealth mode
    set_up_stealth(sb.driver)

    if not sb.is_text_visible("Username", '[for="user_login"]'):
        sb.driver.uc_open_with_reconnect(url, 4)

    sb.assert_text("Username", '[for="user_login"]', timeout=555)
    sb.assert_element('label[for="user_login"]')
    sb.highlight('button:contains("Sign in")')
    sb.highlight('h1:contains("GitLab.com")')
    sb.post_message("SeleniumBase wasn't detected", duration=555)

    # Verify the page after refreshing
    sb.assert_text("Username", '[for="user_login"]', timeout=55)
    sb.assert_element('label[for="user_login"]')
    sb.highlight('button:contains("Sign in")')
    sb.highlight('h1:contains("GitLab.com")')
    sb.post_message("SeleniumBase wasn't detected", duration=55)

[mdmintz]

I don't personally know how to mask a geolocation without the browser indicating that the fingerprint is being masked from it. UC Mode makes your browser appear human-controlled under default settings, but if you change your fingerprint (eg. change the browser's actual geolocation), then I'm not sure how to keep the browser from indicating that the geolocation has been changed.

[misterpeople]

I see. Do you have any suggestions for another library or another way I can check? Technically, setting my PC to the required timezone won't solve this issue entirely?

I guess my only option is the script below. With this script, I was able to bypass all PixelScan tests and sometimes the login at https://gitlab.com/users/sign_in. Do you have any idea why this is happening? I am treating both websites the same way and using a high-quality proxy provider. Will clicking the "Verify you are a human" button with SeleniumBase help in any way?

    import os
from seleniumbase import SB
import logging

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# Proxy settings
PROXY_HOST = "demo"
PROXY_PORT = "demo"
PROXY_USER = "demo"
PROXY_PASS = "demo"

# Construct the proxy URL
proxy_url = f"{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}"

def set_timezone(sb):
    tz_params = {'timezoneId': 'Asia/Kolkata'}
    sb.driver.execute_cdp_cmd('Emulation.setTimezoneOverride', tz_params)
    logging.info("Set timezone to Asia/Kolkata")

with SB(uc=True, test=True, proxy=proxy_url) as sb:
    # Open PixelScan in the first tab
    sb.driver.get("https://pixelscan.net/")
    set_timezone(sb)
    sb.refresh()  # Refresh the page after setting the timezone
    sb.remove_elements("jdiv")  # Remove chat widgets
    sb.assert_text("No automation framework detected", "pxlscn-bot-detection")
    not_masking = "You are not masking your fingerprint"
    sb.assert_text(not_masking, "pxlscn-fingerprint-masking")
    sb.highlight("span.text-success", loops=8)
    sb.sleep(1)
    sb.highlight("pxlscn-fingerprint-masking div", loops=9, scroll=False)
    sb.sleep(1)
    sb.highlight("div.bot-detection-context", loops=10, scroll=False)
    sb.sleep(2)

    # Open GitLab in a new tab
    sb.driver.execute_script("window.open('https://gitlab.com/users/sign_in', '_blank');")
    sb.driver.switch_to.window(sb.driver.window_handles[1])
    set_timezone(sb)
    sb.refresh()  # Refresh the page after setting the timezone
    sb.assert_text("Username", '[for="user_login"]', timeout=99)
    sb.assert_element('label[for="user_login"]')
    sb.highlight('button:contains("Sign in")')
    sb.highlight('h1:contains("GitLab.com")')
    sb.post_message("SeleniumBase wasn't detected", duration=99)

[mdmintz]

I would experiment and see if you can figure it out. I’ve given you as much information as I can.

[misterpeople]

Sure. Thank you! if anything pops up and you find a solution or anyone else please inform me 🙏

…

On Mon, Jun 3, 2024, 00:07 Michael Mintz ***@***.***> wrote: I would experiment and see if you can figure it out. I’ve given you as much information as I can. — Reply to this email directly, view it on GitHub <#2818 (reply in thread)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/ASIE6WN2LX26BUVTL5AWIWTZFOCS7AVCNFSM6AAAAABIT44TS2VHI2DSMVQWIX3LMV43SRDJONRXK43TNFXW4Q3PNVWWK3TUHM4TMNBQHE4DS> . You are receiving this because you authored the thread.Message ID: ***@***.*** com>