From c1a895800a59cae81a9c6c8087a71613ea8ad095 Mon Sep 17 00:00:00 2001 From: spectrem12 Date: Thu, 23 Jul 2020 02:42:18 -0500 Subject: [PATCH 1/3] Added some more debugging logs for understanding the questions as they are bieng asked. Added time to the logs. Added more random sleeps. There is an issue when there are multiple text fields that Im only able to grab the first one or all of them. Not sure how to just select the exact one I need based upon the question being asked. --- easyapplybot.py | 60 +++++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/easyapplybot.py b/easyapplybot.py index d0d67d1..a78573b 100644 --- a/easyapplybot.py +++ b/easyapplybot.py @@ -190,7 +190,7 @@ def applications_loop(self, position, location): # get easy apply button button = self.get_easy_apply_button() - if button is not False: + if button : log.info("It appears that the apply button is considered an EASY apply") #TODO Need to confirm that its an easy apply button by checking the URL is still linkedin URL and not a redirect string_easy = "* has Easy Apply Button" @@ -216,7 +216,7 @@ def applications_loop(self, position, location): result = False position_number = str(count_job + jobs_per_page) - log.info(f"Position {position_number}:\n {self.browser.title} \n {string_easy} \n") + log.info(f"\nSuccess?: {result} \n Position {position_number}\n {self.browser.title} \n {string_easy} \n {job}") self.write_to_file(button, jobID, self.browser.title, result) @@ -295,11 +295,12 @@ def is_present(button_locator): try: - time.sleep(3) + time.sleep(random.uniform(2.2, 4.3)) log.info("Attempting to send resume") - #TODO These locators are not future proof. These labels could easily change. Ideally we would search for contained text; + #TODO These locators are not future proof. These labels could easily change. + # Ideally we would search for contained text; # was unable to get it to work using XPATH and searching for contained text - upload_locator = (By.CSS_SELECTOR, "label[aria-label='DOC, DOCX, PDF formats only (2 MB).']") + upload_locator = (By.CSS_SELECTOR, "input[name='file']") next_locator = (By.CSS_SELECTOR, "button[aria-label='Continue to next step']") review_locator = (By.CSS_SELECTOR, "button[aria-label='Review your application']") submit_locator = (By.CSS_SELECTOR, "button[aria-label='Submit application']") @@ -321,6 +322,8 @@ def is_present(button_locator): # Upload Cover Letter if possible + #TODO Should check if there is already a resume that is saved from the last time the application was attempted. + # If so, then remove and re upload it in case there is new version. if is_present(upload_locator): input_buttons = self.browser.find_elements(upload_locator[0], @@ -336,15 +339,20 @@ def is_present(button_locator): #input_button[0].send_keys(self.cover_letter_loctn) - time.sleep(random.uniform(4.5, 6.5)) - for i, button_locator in enumerate( [upload_locator, next_locator, review_locator, submit_locator, submit_application_locator]): + #Sleep every iteration so that the bot is harded to detect. + time.sleep(random.uniform(2.2, 4.3)) + log.info("Searching for button locator: %s", str(button_locator)) if is_present(button_locator): log.info("button found with this locator: %s", str(button_locator)) - button = self.wait.until(EC.element_to_be_clickable(button_locator)) + try: + button = self.wait.until(EC.element_to_be_clickable(button_locator)) + except TimeoutException: + log.exception("Timed out waiting for button %s ", button_locator) + continue else: log.info("Unable to find button locator: %s", str(button_locator)) continue @@ -366,7 +374,7 @@ def is_present(button_locator): try: log.info("Found test element %s", testLabel_locator) text = testLabelElement.text - log.info("test element text: %s", text) + log.warning("Question Text: %s", text) #assuming this question is asking if I am authorized to work in the US if ("Are you" in text and "authorized" in text) or ("Have You" in text and "education" in text): #Be sure to find the child element of the current test question section @@ -377,7 +385,7 @@ def is_present(button_locator): log.info("Clicked the radio button %s", yes_locator) #assuming this question is asking if I require sponsorship - if "require" in text and "sponsorship" in text: + elif "require" in text and "sponsorship" in text: noRadio = testLabelElement.find_element(By.XPATH, no_locator[1]) time.sleep(1) log.info("Attempting to click the radio button for %s", no_locator) @@ -385,7 +393,15 @@ def is_present(button_locator): log.info("Clicked the radio button %s", no_locator) # assuming this question is asking if I have a Bachelor's degree - if "you have" in text and "Bachelor's" in text: + elif (("You have" in text) or ("Have you" in text)) and "Bachelor's" in text: + yesRadio = testLabelElement.find_element(By.XPATH, yes_locator[1]) + time.sleep(1) + log.info("Attempting to click the radio button for %s", yes_locator) + self.browser.execute_script("arguments[0].click()", yesRadio) + log.info("Clicked the radio button %s", yes_locator) + + # assuming this question is asking if I have a Master's degree + elif (("You have" in text) or ("Have you" in text)) and "Master's" in text: yesRadio = testLabelElement.find_element(By.XPATH, yes_locator[1]) time.sleep(1) log.info("Attempting to click the radio button for %s", yes_locator) @@ -396,7 +412,7 @@ def is_present(button_locator): #TODO Need to add a configuration file with all the answer for these questions versus having them hardcoded. #Some questions are asking how many years of experience you have in a specific skill #Automatically put the number of years that I have worked. - if "How many years" in text and "experience" in text: + elif "How many years" in text and "experience" in text: textField = testLabelElement.find_element(By.XPATH, textInput_locator[1]) time.sleep(1) log.info("Attempting to click the text field for %s", textInput_locator) @@ -408,7 +424,7 @@ def is_present(button_locator): log.info("Sent keys to the text field %s", textInput_locator) #This should be updated to match the language you speak. - if "Do you" in text and "speak" in text: + elif "Do you" in text and "speak" in text: if "English" in text: yesRadio = testLabelElement.find_element(By.XPATH, yes_locator[1]) time.sleep(1) @@ -423,7 +439,8 @@ def is_present(button_locator): self.browser.execute_script("arguments[0].click()", noRadio) log.info("Clicked the radio button %s", no_locator) - + else: + log.warning("Unable to find question in my tiny database") except Exception as e: log.exception("Could not answer additional questions: %s", e) @@ -440,7 +457,7 @@ def is_present(button_locator): if button_locator == upload_locator: log.info("Uploading resume now") - time.sleep(2) + time.sleep(random.uniform(2.2, 4.3)) driver.execute_script("arguments[0].click()", button) #TODO This can only handle Chrome right now. Firefox or other browsers will need to be handled separately @@ -452,13 +469,14 @@ def is_present(button_locator): wsh.SendKeys(str(self.resume_loctn)) time.sleep(1) wsh.SendKeys("{ENTER}") + log.info("Just finished using button %s ", button_locator) else: try: log.info("attempting to click button: %s", str(button_locator)) response = button.click() if (button_locator == submit_locator) or (button_locator == submit_application_locator): - log.info("Clicked the submit button. RESPONSE: %s", str(response)) + log.info("Clicked the submit button.") submitted = True return submitted except EC.StaleElementReferenceException: @@ -468,9 +486,7 @@ def is_present(button_locator): log.warning("Unable to submit. It appears none of the buttons were found.") break - randoTime = random.uniform(1.5, 2.5) - log.info("Just finished using button %s ; Im going to sleep for %s ;", str(button_locator), randoTime) - time.sleep(randoTime) + # After submitting the application, a dialog shows up, we need to close this dialog close_button_locator = (By.CSS_SELECTOR, "button[aria-label='Dismiss']") @@ -528,12 +544,12 @@ def setupLogger(): if not os.path.isdir('./logs'): os.mkdir('./logs') - logging.basicConfig(filename=('./logs/' + str(dt)+'applyJobs.log'), filemode='w', format='%(name)s::%(levelname)s::%(message)s', datefmt='./logs/%d-%b-%y %H:%M:%S') #TODO need to check if there is a log dir available or not - + # TODO need to check if there is a log dir available or not + logging.basicConfig(filename=('./logs/' + str(dt)+'applyJobs.log'), filemode='w', format='%(asctime)s::%(name)s::%(levelname)s::%(message)s', datefmt='./logs/%d-%b-%y %H:%M:%S') log.setLevel(logging.DEBUG) c_handler = logging.StreamHandler() c_handler.setLevel(logging.DEBUG) - c_format = logging.Formatter('%(name)s::%(levelname)s::%(lineno)d- %(message)s') + c_format = logging.Formatter('%(asctime)s::%(name)s::%(levelname)s::%(lineno)d- %(message)s') c_handler.setFormatter(c_format) log.addHandler(c_handler) From d98b3c013ff1eaaed14255c7d1882b5cbdff21d4 Mon Sep 17 00:00:00 2001 From: spectrem12 Date: Fri, 24 Jul 2020 02:51:04 -0500 Subject: [PATCH 2/3] Updated naming to question elements. Updated XPATH searches. Now enters alpha numeric values in the correct text field. --- easyapplybot.py | 68 ++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/easyapplybot.py b/easyapplybot.py index a78573b..af427c4 100644 --- a/easyapplybot.py +++ b/easyapplybot.py @@ -1,25 +1,25 @@ -import time, random, os, csv, platform -from selenium.webdriver.chrome.options import Options -from selenium.common.exceptions import TimeoutException -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.common.by import By -from selenium.webdriver.support import expected_conditions as EC -from selenium import webdriver - -from bs4 import BeautifulSoup -import pandas as pd -import pyautogui - -from urllib.request import urlopen -from webdriver_manager.chrome import ChromeDriverManager -import re -import yaml +import csv import json -from datetime import datetime, timedelta import logging +import os +import random +import re +import time +from datetime import datetime, timedelta +import pandas as pd +import pyautogui import win32com.client as comctl +import yaml +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.common.exceptions import TimeoutException +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait +from webdriver_manager.chrome import ChromeDriverManager wsh = comctl.Dispatch("WScript.Shell") @@ -309,10 +309,10 @@ def is_present(button_locator): cover_letter = (By.CSS_SELECTOR, "input[name='file']") - testLabel_locator = (By.XPATH, "//span[@data-test-form-element-label-title='true']") - yes_locator = (By.XPATH, "//input[@value='Yes']") - no_locator = (By.XPATH, "//input[@value='No']") - textInput_locator = (By.XPATH, "//input[@type='text']") + question_locator = (By.XPATH, ".//div[@class='jobs-easy-apply-form-section__grouping']") + yes_locator = (By.XPATH, ".//input[@value='Yes']") + no_locator = (By.XPATH, ".//input[@value='No']") + textInput_locator = (By.XPATH, ".//input[@type='text']") submitted = False @@ -368,17 +368,17 @@ def is_present(button_locator): #TODO these questions will need to be logged so that way, individuals can look through the logs and add them at the end of an application run. #Required question expects an answer. Search through possible questions/answer combos - if is_present(testLabel_locator) and attemptQuestions: - for testLabelElement in self.browser.find_elements(testLabel_locator[0], - testLabel_locator[1]): + if is_present(question_locator) and attemptQuestions: + questionSections = self.browser.find_elements(question_locator[0], question_locator[1]) + for questionElement in questionSections: try: - log.info("Found test element %s", testLabel_locator) - text = testLabelElement.text + log.info("Found test element %s", questionElement) + text = questionElement.text log.warning("Question Text: %s", text) #assuming this question is asking if I am authorized to work in the US if ("Are you" in text and "authorized" in text) or ("Have You" in text and "education" in text): #Be sure to find the child element of the current test question section - yesRadio = testLabelElement.find_element(By.XPATH, yes_locator[1]) + yesRadio = questionElement.find_element(By.XPATH, yes_locator[1]) time.sleep(1) log.info("Attempting to click the radio button for %s", yes_locator) self.browser.execute_script("arguments[0].click()", yesRadio) @@ -386,7 +386,7 @@ def is_present(button_locator): #assuming this question is asking if I require sponsorship elif "require" in text and "sponsorship" in text: - noRadio = testLabelElement.find_element(By.XPATH, no_locator[1]) + noRadio = questionElement.find_element(By.XPATH, no_locator[1]) time.sleep(1) log.info("Attempting to click the radio button for %s", no_locator) self.browser.execute_script("arguments[0].click()", noRadio) @@ -394,7 +394,7 @@ def is_present(button_locator): # assuming this question is asking if I have a Bachelor's degree elif (("You have" in text) or ("Have you" in text)) and "Bachelor's" in text: - yesRadio = testLabelElement.find_element(By.XPATH, yes_locator[1]) + yesRadio = questionElement.find_element(By.XPATH, yes_locator[1]) time.sleep(1) log.info("Attempting to click the radio button for %s", yes_locator) self.browser.execute_script("arguments[0].click()", yesRadio) @@ -402,7 +402,7 @@ def is_present(button_locator): # assuming this question is asking if I have a Master's degree elif (("You have" in text) or ("Have you" in text)) and "Master's" in text: - yesRadio = testLabelElement.find_element(By.XPATH, yes_locator[1]) + yesRadio = questionElement.find_element(By.XPATH, yes_locator[1]) time.sleep(1) log.info("Attempting to click the radio button for %s", yes_locator) self.browser.execute_script("arguments[0].click()", yesRadio) @@ -413,7 +413,7 @@ def is_present(button_locator): #Some questions are asking how many years of experience you have in a specific skill #Automatically put the number of years that I have worked. elif "How many years" in text and "experience" in text: - textField = testLabelElement.find_element(By.XPATH, textInput_locator[1]) + textField = questionElement.find_element(By.XPATH, textInput_locator[1]) time.sleep(1) log.info("Attempting to click the text field for %s", textInput_locator) self.browser.execute_script("arguments[0].click()", textField) @@ -426,14 +426,14 @@ def is_present(button_locator): #This should be updated to match the language you speak. elif "Do you" in text and "speak" in text: if "English" in text: - yesRadio = testLabelElement.find_element(By.XPATH, yes_locator[1]) + yesRadio = questionElement.find_element(By.XPATH, yes_locator[1]) time.sleep(1) log.info("Attempting to click the radio button for %s", yes_locator) self.browser.execute_script("arguments[0].click()", yesRadio) log.info("Clicked the radio button %s", yes_locator) #if not english then say no. else: - noRadio = testLabelElement.find_element(By.XPATH, no_locator[1]) + noRadio = questionElement.find_element(By.XPATH, no_locator[1]) time.sleep(1) log.info("Attempting to click the radio button for %s", no_locator) self.browser.execute_script("arguments[0].click()", noRadio) From 8f287e3c96e52cbd3cc7439854c9f27fda383c31 Mon Sep 17 00:00:00 2001 From: spectrem12 Date: Fri, 24 Jul 2020 03:43:36 -0500 Subject: [PATCH 3/3] Leaving cover letter uploading locator in to be fixed later. --- easyapplybot.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/easyapplybot.py b/easyapplybot.py index af427c4..d7f3581 100644 --- a/easyapplybot.py +++ b/easyapplybot.py @@ -296,11 +296,11 @@ def is_present(button_locator): try: time.sleep(random.uniform(2.2, 4.3)) - log.info("Attempting to send resume") + log.info("Attempting to apply") #TODO These locators are not future proof. These labels could easily change. # Ideally we would search for contained text; # was unable to get it to work using XPATH and searching for contained text - upload_locator = (By.CSS_SELECTOR, "input[name='file']") + upload_locator = (By.CSS_SELECTOR, "label[aria-label='DOC, DOCX, PDF formats only (2 MB).']") next_locator = (By.CSS_SELECTOR, "button[aria-label='Continue to next step']") review_locator = (By.CSS_SELECTOR, "button[aria-label='Review your application']") submit_locator = (By.CSS_SELECTOR, "button[aria-label='Submit application']") @@ -308,7 +308,6 @@ def is_present(button_locator): error_locator = (By.CSS_SELECTOR, "p[data-test-form-element-error-message='true']") cover_letter = (By.CSS_SELECTOR, "input[name='file']") - question_locator = (By.XPATH, ".//div[@class='jobs-easy-apply-form-section__grouping']") yes_locator = (By.XPATH, ".//input[@value='Yes']") no_locator = (By.XPATH, ".//input[@value='No']") @@ -320,14 +319,14 @@ def is_present(button_locator): while not submitted: button = None - # Upload Cover Letter if possible + # Upload if possible #TODO Should check if there is already a resume that is saved from the last time the application was attempted. # If so, then remove and re upload it in case there is new version. if is_present(upload_locator): - - input_buttons = self.browser.find_elements(upload_locator[0], - upload_locator[1]) + log.info("Resume upload option available. Attempting to upload.") + input_buttons = self.browser.find_elements(cover_letter[0], + cover_letter[1]) for input_button in input_buttons: parent = input_button.find_element(By.XPATH, "..") sibling = parent.find_element(By.XPATH, "preceding-sibling::*") @@ -336,9 +335,6 @@ def is_present(button_locator): if key in sibling.text or key in grandparent.text: input_button.send_keys(self.uploads[key]) - - #input_button[0].send_keys(self.cover_letter_loctn) - for i, button_locator in enumerate( [upload_locator, next_locator, review_locator, submit_locator, submit_application_locator]):