diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..e94c812
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+/venv
+/build
+# /dist
+*.spec
+/__pycache__
\ No newline at end of file
diff --git a/FormAutomation.py b/FormAutomation.py
new file mode 100644
index 0000000..42e7c6d
--- /dev/null
+++ b/FormAutomation.py
@@ -0,0 +1,582 @@
+import time
+import numpy as np
+from pyautogui import screenshot, locateAll, locateAllOnScreen, click, moveTo, size, scroll, move , confirm
+from requests import post
+
+from ImageProcessing import get_image_lines
+
+# box is a tuple of 4 values: (left, top, width, height)
+# ((x1,y1), (x2,y2).... (x3,x4))
+
+
+
+class Box:
+ def __init__(self, box) -> None:
+ # when l, t, w, h
+ if isinstance(box, tuple) and len(box) == 4 and all(isinstance(i, (int, float)) for i in box):
+ self.l, self.t, self.w, self.h = box
+ self.vertices = ((self.l, self.t), (self.l + self.w, self.t),
+ (self.l + self.w, self.t + self.h), (self.l, self.t + self.h))
+ self.centre = (self.l + self.w / 2, self.t + self.h / 2)
+ self.box_tuple = box
+ # when vertices
+ elif isinstance(box, tuple) and len(box) == 4 and all(isinstance(i, tuple) and len(i) == 2 for i in box):
+ self.vertices = box
+ self.l, self.t = box[0]
+ self.w = box[1][0] - box[0][0]
+ self.h = box[2][1] - box[1][1]
+ self.box_tuple = (self.l, self.t, self.w, self.h)
+ self.centre = (self.l + self.w / 2, self.t + self.h / 2)
+ else:
+ raise ValueError("Invalid input format for Box")
+ def __str__(self) -> str:
+ return f"{self.box_tuple}"
+ def __repr__(self) -> str:
+ return f"{self.box_tuple}"
+ def is_within(self, box):
+ return self.l >= box.l and self.t >= box.t and self.l + self.w <= box.l + box.w and self.t + self.h <= box.t + box.h
+
+class ScreenShot:
+ def __init__(self, box, file_path=None) -> None:
+ self.box = Box(box)
+ self.ss = screenshot(region=(self.box.l, self.box.t, self.box.w, self.box.h), imageFilename=file_path).convert('RGB')
+ self.gray_ss = self.ss.convert('L')
+ self.path = file_path
+
+ # self.screen_size = size()
+
+ def get_pixel(self, x, y):
+ return self.ss.getpixel((x-self.box.l, y-self.box.t))
+ def get_gray_pixel(self, x, y):
+ return self.gray_ss.getpixel((x-self.box.l, y-self.box.t))
+
+ def is_line_pixel(self, x, y, direction):
+
+ if self.get_gray_pixel(x, y) > 230:
+ return False
+
+ if direction == 'horizontal':
+ line_length = 20
+ bool_list = []
+ for i in range(line_length):
+ if(x+i >= self.box.l + self.box.w):
+ break
+ bool_list.append(abs(self.get_gray_pixel(x+i, y) - self.get_gray_pixel(x, y)) < 1)
+
+ return all(bool_list)
+
+ else:
+ line_length = 20
+ bool_list = []
+ for i in range(line_length):
+ if(y+i >= self.box.t + self.box.h):
+ break
+ bool_list.append(abs(self.get_gray_pixel(x, y+i) - self.get_gray_pixel(x, y)) < 1)
+ return all(bool_list)
+
+
+ def find_all_img(self, image, confidence=0.9):
+ # try:
+ try:
+ found_coords = list(locateAll(needleImage=image, haystackImage=self.ss, confidence=confidence))
+ except:
+ return []
+
+ res = []
+ for coord in found_coords:
+ x, y, w, h = coord
+ temp_tuple = (int(x + self.box.l), int(y + self.box.t), int(w), int(h))
+ box = Box(temp_tuple)
+ res.append(box)
+ return res
+
+
+ def find_text(self, text_to_find, texts_all):
+ for text in texts_all:
+ if text["description"].lower() == text_to_find.lower():
+ vertices = text["boundingPoly"]["vertices"]
+ box_vertices = tuple((vertex['x']+self.box.l, vertex['y']+self.box.t) for vertex in vertices)
+
+ return Box(box_vertices)
+ return None
+ def find_text_in_bounds(self, text_to_find, bound_box, texts_all):
+ for text in texts_all:
+ if text["description"] == text_to_find:
+ text_box = Box(tuple((vertex['x']+self.box.l, vertex['y']+self.box.t) for vertex in text["boundingPoly"]["vertices"]))
+ if text_box.is_within(bound_box):
+ return text_box
+ return None
+
+ def find_all_text(self, text_to_find, texts_all):
+ found_text_coords = []
+ for text in texts_all:
+ if text["description"] == text_to_find:
+ vertices = Box((vertex['x']+self.box.l, vertex['y']+self.box.t) for vertex in text["boundingPoly"]["vertices"])
+ found_text_coords.append(vertices)
+ return found_text_coords
+
+ def find_all_text_in_bounds(self, text_to_find, bound_box, texts_all):
+ found_text_coords = []
+ for text in texts_all:
+ if text["description"] == text_to_find:
+ text_box = Box((vertex['x']+self.box.l, vertex['y']+self.box.t) for vertex in text["boundingPoly"]["vertices"])
+ if text_box.is_within(bound_box):
+ found_text_coords.append(text_box)
+ return found_text_coords
+
+ def find_texts(self, texts_to_find:list, texts_all):
+ found_texts = {}
+ for text in texts_all:
+ if text["description"] in texts_to_find:
+ vertices = Box(tuple((vertex['x']+self.box.l, vertex['y']+self.box.t) for vertex in text["boundingPoly"]["vertices"]))
+ if not found_texts.get(text["description"]):
+ found_texts[text["description"]] = [vertices]
+ else:
+ found_texts[text["description"]].append(vertices)
+ # found_texts[text["description"]] = vertices
+ return found_texts
+
+ def get_lines(self):
+ lines_found = get_image_lines(self.ss)
+
+ # lines_found[0] += self.box.t
+ # lines_found[1] += self.box.l
+ return (lines_found[0] + self.box.t, lines_found[1] + self.box.l)
+
+
+class Form:
+ def __init__(self, heading:str, heading_box:Box, border_box:Box) -> None:
+ self.heading = heading
+ self.heading_box = heading_box
+ self.border_box = border_box
+
+ self.buttons = []
+ self.selected = None
+ self.button_path = "button.png"
+ self.selected_button_path = "button_selected.png"
+
+ self.ss = None
+ self.ss = ScreenShot(self.border_box.box_tuple, f"form_{self.heading}.png")
+ # self.selected_button_path =
+
+ def establish_borders(self, screen_ss:ScreenShot):
+ # Find bottom border
+ for y in range(self.heading_box.t + self.heading_box.h, screen_ss.box.t + screen_ss.box.h):
+ if screen_ss.is_line_pixel(self.heading_box.l + 1, y, 'horizontal'):
+ self.border_box.h = y - self.heading_box.t
+ break
+
+ # Find right border
+ for x in range(self.heading_box.l + self.heading_box.w, screen_ss.box.l + screen_ss.box.w):
+ if screen_ss.is_line_pixel(x, self.heading_box.t, 'vertical'):
+ self.border_box.w = x - self.heading_box.l
+ break
+ new_box = Box((self.border_box.l, self.border_box.t, self.border_box.w, self.border_box.h))
+ self.border_box = new_box
+ self.ss = ScreenShot(self.border_box.box_tuple, f"form_{self.heading}.png")
+
+ def find_buttons(self, confidence):
+
+ try:
+ self.buttons += self.ss.find_all_img(self.button_path, confidence)
+ except:
+ pass
+
+ try:
+ selected_found = self.ss.find_all_img(self.selected_button_path, confidence)
+ self.buttons += selected_found
+ self.selected = selected_found[0]
+ except:
+ pass
+
+ # remove buttons which are within pixels of each other
+ if(len(self.buttons) == 0):
+ return self.buttons
+ self.buttons = [self.buttons[0]] + [self.buttons[i] for i in range(1, len(self.buttons)) if not any(abs(self.buttons[i].l - self.buttons[j].l) < 10 and abs(self.buttons[i].t - self.buttons[j].t) < 10 for j in range(i))]
+
+
+ # self.buttons += self.ss.find_all_img("button_selected.png", confidence)
+ self.buttons = sort_boxes(self.buttons)
+ return self.buttons
+
+ def click_button(self, button_index):
+ moveTo(self.buttons[button_index].centre)
+ click()
+ moveTo(100, 100)
+
+ def unselect_button(self):
+ if self.selected:
+ moveTo(self.selected.centre)
+ click()
+ moveTo(100, 100)
+ def click_all_buttons(self):
+ for button in self.buttons:
+ moveTo(button.centre)
+ # click()
+ # time.sleep(1)
+
+def pix_within_threshold(pix1, pix2, threshold):
+ return all(abs(pix1[i] - pix2[i]) < threshold for i in range(3))
+def find_all_img_in_bounds(image, bound_box, confidence):
+
+ ss_in_bounds = ScreenShot(bound_box.box_tuple, "temp_ss.png")
+ found_coords = ss_in_bounds.find_all_img(image, confidence)
+ return found_coords
+
+
+# array of boxes, should be smallest x value first and smallest y value first
+def sort_boxes(boxes):
+ boxes.sort(key=lambda x: x.l)
+ boxes.sort(key=lambda x: x.t)
+ return boxes
+
+def get_legit_headings(found):
+ legit_found = {}
+ for heading, verts in found.items():
+ for box in verts:
+ if not any(screen_ss.is_line_pixel(box.l, y, 'horizontal') for y in range(box.t, box.t -8, -1)):
+ # print(f"skipping {heading} because of {box}")
+ continue
+ legit_found[heading] = box
+ break
+
+ return legit_found
+
+server_url = 'https://googlecloudapi.vercel.app/api/v1'
+
+def get_annotations(ss:ScreenShot):
+ with open(ss.path, 'rb') as image:
+ response = post(f'{server_url}/image/annotations', files={'file': image})
+ return response.json()
+
+
+def num_within_threshold(num, target, threshold):
+ return abs(num - target) < threshold
+
+if __name__ == "__main__":
+ screen_size = size()
+ top_bar = 120
+ bottom_bar = 80
+
+
+ # lines = screen_ss.get_lines()
+ # print(lines)
+ # horizontals, verticals = lines
+
+ headings_options_count = {
+ "M0069" : 2,
+ "A1110B": 3,
+
+ "M0080": 4,
+ "M0100": 8,
+ "M0110":4,
+
+ "HCS_0110_Facility": 2,
+ # "Blood Pressure": 2,
+ "B0200":5,
+ "B1000":6,
+ "B1300":7,
+
+ "C0100":3,
+ "C0200":4,
+ "C0300A":5,
+ "C0300B":4,
+ "C0300C":4,
+ "C0400A":4,
+ "C0400B":4,
+ "C0400C":4,
+
+ "C1310A":3,
+ "C1310B":4,
+ "C1310C":4,
+ "C1310D":4,
+
+ "M1700":5,
+ "M1710":6,
+ "M1720":5,
+
+ "D0150A1":4,
+ "D0150A2":4,
+
+ "D0150B1":4,
+ "D0150B2":4,
+
+ "D0150C1":4,
+ "D0150C2":4,
+
+ "D0150D1":4,
+ "D0150D2":4,
+
+ "D0150E1":4,
+ "D0150E2":4,
+
+ "D0150F1":4,
+ "D0150F2":4,
+
+ "D0150G1":4,
+ "D0150G2":4,
+
+ "D0150H1":4,
+ "D0150H2":4,
+
+ "D0150I1":4,
+ "D0150I2":4,
+
+ "M1740":3,
+ "M1745":6,
+
+ "M2102_CARE_TYPE_SRC_SPRVSN":5,
+
+ "M1800":4,
+ "M1810":4,
+ "M1820":4,
+ "M1830":4,
+ "M1840":4,
+ "M1845":4,
+ "M1850":6,
+ "M1860":7,
+
+ "GG0170A1":11,
+ "GG0170A2":11,
+ "GG0170B1":11,
+ "GG0170B2":11,
+ "GG0170C_MOBILITY_SOCROC_PERF":11,
+ "GG0170C_MOBILITY_DSCHG_GOAL":11,
+ "GG0170D1":11,
+ "GG0170D2":11,
+ "GG0170E1":11,
+ "GG0170E2":11,
+ "GG0170F1":11,
+ "GG0170F2":11,
+ "GG0170G1":11,
+ "GG0170G2":11,
+ "GG0170I1":11,
+ "GG0170I2":11,
+ "GG0170J1":11,
+ "GG0170J2":11,
+ "GG0170K1":11,
+ "GG0170K2":11,
+ "GG0170L1":11,
+ "GG0170L2":11,
+ "GG0170M1":11,
+ "GG0170M2":11,
+ # "GG0170N1":11,
+ "GG0170N2":11,
+ # "GG0170O1":11,
+ "GG0170O2":11,
+ "GG0170P1":11,
+ "GG0170P2":11,
+ "GG0170Q1":11,
+
+ "M1600":4,
+ "M1610":3,
+ "M1620":8,
+ "M1630":3,
+
+ "M1400":5,
+
+ "M1306":2,
+
+ "M1322":5,
+ "M1324":5,
+ "M1330":4,
+ "M1332":4,
+ "M1334":3,
+ "M1340":3,
+ "M1342":4,
+
+ "J0510":6,
+ "J0520":6,
+ "J0530":5,
+
+ "K0520A1":3,
+ "K0520B1":3,
+ "K0520C1":3,
+ "K0520D1":3,
+ "K0520Z1":3,
+
+ "M1870":6,
+
+ "N0415A1":3,
+ "N0415A2":3,
+ "N0415E1":3,
+ "N0415E2":3,
+ "N0415F1":3,
+ "N0415F2":3,
+ "N0415H1":3,
+ "N0415H2":3,
+ "N0415I1":3,
+ "N0415I2":3,
+ "N0415J1":3,
+ "N0415J2":3,
+ "N0415Z1":3,
+
+ "M2001":3,
+ "M2003":2,
+ "M2010":3,
+ "M2020":5,
+ "M2030":5,
+
+ "00110A1a":3,
+ "00110A2a":3,
+ "00110A3a":3,
+ "00110A10a":3,
+ "00110B1a":3,
+ "00110C1a":3,
+ "00110C2a":3,
+ "00110C3a":3,
+ "00110C4a":3,
+ "00110D1a":3,
+ "00110D2a":3,
+ "00110D3a":3,
+ "00110E1a":3,
+ "00110F1a":3,
+ "00110G1a":3,
+ "00110G2a":3,
+ "00110G3a":3,
+ "00110H1a":3,
+ "00110H2a":3,
+ "00110H3a":3,
+ "00110H4a":3,
+ "00110H10a":3,
+ "00110I1a":3,
+ "00110J1a":3,
+ "00110J2a":3,
+ "00110J3a":3,
+
+ "0011001a":3,
+ "0011002a":3,
+ "0011003a":3,
+ "0011004a":3,
+ "00110Z1a":3,
+ }
+
+ headings_choices = {
+
+ "M0100": 5,
+ "M0069": 1,
+
+ # "B0200":0,
+ "C1310A":0,
+ "C1310B":1,
+ "C1310C":2,
+ "C1310D":3,
+
+ "M1700":2,
+ "M1710":3,
+ "M1720":1,
+
+ "D0150A1":0,
+ "D0150A2":1,
+ "D0150B1":3,
+ "D0150B2":2,
+ "D0150C1":1,
+ "D0150C2":0,
+ "D0150D1":0,
+ "D0150D2":3,
+ "D0150E1":2,
+ "D0150E2":1,
+ "D0150F1":0,
+ # "D0150F2":0
+ "D0700":5,
+ "M2102_CARE_TYPE_SRC_SPRVSN":2,
+
+ "M1800": 3,
+ "M1810": 1,
+ "M1820": 2,
+ "M1830": 0,
+ "M1840": 0,
+ "M1845": 3,
+ "M1850": 1,
+ "M1860": 2,
+ "B0200":0,
+ "B1000":1,
+ "B1300":1,
+ "C0300A":0,
+ "C0300B":1,
+ "C0300C":1,
+ "M0150":1,
+ "GG0100": 0,
+ "GG0110": 1,
+ "GG0130A": 0,
+ }
+
+ # debugging stuff:
+ found_text = set()
+ found_legit = set()
+ incorrect_buttons = set()
+
+ problem_paths = ["problem_1.png", "problem_2.png"]
+ remaining_headings = list(headings_options_count.keys())
+ while len(remaining_headings) > 0:
+ screen_ss = ScreenShot((0, top_bar, screen_size[0], screen_size[1]-top_bar-bottom_bar), "first_ss.png")
+ problems = []
+ for path in problem_paths:
+
+ problems += screen_ss.find_all_img(path, 0.9)
+ for problem in problems:
+ moveTo(problem.centre)
+ click()
+ time.sleep(1)
+ moveTo(100, screen_size[1]/2)
+
+ annotations = get_annotations(screen_ss)["data"]["textAnnotations"]
+ end = screen_ss.find_text("Rehab", annotations)
+
+ if(end):
+ print("End found")
+ break
+
+ found = screen_ss.find_texts(headings_options_count.keys(), annotations)
+
+ (found_text.add(head) for head in found.keys())
+
+ found = get_legit_headings(found)
+
+ (found_legit.add(head) for head in found.keys())
+
+ # headings_boxes = found
+ print("heading text boxes:", found)
+ # screen_ss = ScreenShot((0, 120, screen_size[0], screen_size[1]-120-40), "first_ss.png")
+ # populate forms on screen:
+ for heading, box in found.items():
+ if(heading not in remaining_headings):
+ continue
+ box = found[heading]
+
+ form_box = Box((
+ box.l,
+ box.t,
+ int(abs(screen_size[0] - box.l)),
+ int(abs(screen_size[1] - box.t))
+ ))
+
+ form_obj = Form(heading, box, form_box)
+ form_obj.establish_borders(screen_ss)
+ form_obj.find_buttons(0.85)
+ if(len(form_obj.buttons) != headings_options_count[heading]):
+ incorrect_buttons.add(heading)
+ print(f"Incorrect buttons for {heading}: {len(form_obj.buttons)}")
+ continue
+ incorrect_buttons.discard(heading)
+
+ form_obj.unselect_button()
+ form_obj.click_all_buttons()
+ moveTo(100, screen_size[1]/2)
+ # print(form_obj.buttons)
+ # moveTo((right_border, bottom_border))
+ # time.sleep(2)
+ remaining_headings.remove(heading)
+ pass
+
+ scroll(-100)
+
+
+ never_found = set(headings_options_count.keys()) - found_text
+ never_legit = set(headings_options_count.keys()) - found_legit
+
+ print(f"Never found: {never_found}")
+ print(f"Never legit: {never_legit}")
+ print(f"Incorrect buttons: {incorrect_buttons}")
+
+ pass
\ No newline at end of file
diff --git a/FormAutomation_2.py b/FormAutomation_2.py
new file mode 100644
index 0000000..1b6c215
--- /dev/null
+++ b/FormAutomation_2.py
@@ -0,0 +1,593 @@
+import eel
+import time
+import numpy as np
+from pyautogui import screenshot, locateAll, locateAllOnScreen, click, moveTo, size, scroll, move
+from requests import post
+
+eel.init('web')
+
+# Box class definition
+class Box:
+ def __init__(self, box) -> None:
+ # when l, t, w, h
+ if isinstance(box, tuple) and len(box) == 4 and all(isinstance(i, (int, float)) for i in box):
+ self.l, self.t, self.w, self.h = box
+ self.vertices = ((self.l, self.t), (self.l + self.w, self.t),
+ (self.l + self.w, self.t + self.h), (self.l, self.t + self.h))
+ self.centre = (self.l + self.w / 2, self.t + self.h / 2)
+ self.box_tuple = box
+ # when vertices
+ elif isinstance(box, tuple) and len(box) == 4 and all(isinstance(i, tuple) and len(i) == 2 for i in box):
+ self.vertices = box
+ self.l, self.t = box[0]
+ self.w = box[1][0] - box[0][0]
+ self.h = box[2][1] - box[1][1]
+ self.box_tuple = (self.l, self.t, self.w, self.h)
+ self.centre = (self.l + self.w / 2, self.t + self.h / 2)
+ else:
+ raise ValueError("Invalid input format for Box")
+ def __str__(self) -> str:
+ return f"{self.box_tuple}"
+ def __repr__(self) -> str:
+ return f"{self.box_tuple}"
+ def is_within(self, box):
+ return self.l >= box.l and self.t >= box.t and self.l + self.w <= box.l + box.w and self.t + self.h <= box.t + box.h
+
+class ScreenShot:
+ def __init__(self, box, file_path=None) -> None:
+ self.box = Box(box)
+ self.ss = screenshot(region=(self.box.l, self.box.t, self.box.w, self.box.h), imageFilename=file_path).convert('RGB')
+ self.gray_ss = self.ss.convert('L')
+ self.path = file_path
+
+ # self.screen_size = size()
+
+ def get_pixel(self, x, y):
+ return self.ss.getpixel((x-self.box.l, y-self.box.t))
+ def get_gray_pixel(self, x, y):
+ return self.gray_ss.getpixel((x-self.box.l, y-self.box.t))
+
+ def is_line_pixel(self, x, y, direction):
+
+ if self.get_gray_pixel(x, y) > 230:
+ return False
+
+ if direction == 'horizontal':
+ line_length = 20
+ bool_list = []
+ for i in range(line_length):
+ if(x+i >= self.box.l + self.box.w):
+ break
+ bool_list.append(abs(self.get_gray_pixel(x+i, y) - self.get_gray_pixel(x, y)) < 1)
+
+ return all(bool_list)
+
+ else:
+ line_length = 20
+ bool_list = []
+ for i in range(line_length):
+ if(y+i >= self.box.t + self.box.h):
+ break
+ bool_list.append(abs(self.get_gray_pixel(x, y+i) - self.get_gray_pixel(x, y)) < 1)
+ return all(bool_list)
+
+
+ def find_all_img(self, image, confidence=0.9):
+ # try:
+ try:
+ found_coords = list(locateAll(needleImage=image, haystackImage=self.ss, confidence=confidence))
+ except:
+ return []
+
+ res = []
+ for coord in found_coords:
+ x, y, w, h = coord
+ temp_tuple = (int(x + self.box.l), int(y + self.box.t), int(w), int(h))
+ box = Box(temp_tuple)
+ res.append(box)
+ return res
+
+
+ def find_text(self, text_to_find, texts_all):
+ for text in texts_all:
+ if text["description"].lower() == text_to_find.lower():
+ vertices = text["boundingPoly"]["vertices"]
+ box_vertices = tuple((vertex['x']+self.box.l, vertex['y']+self.box.t) for vertex in vertices)
+
+ return Box(box_vertices)
+ return None
+ def find_text_in_bounds(self, text_to_find, bound_box, texts_all):
+ for text in texts_all:
+ if text["description"] == text_to_find:
+ text_box = Box(tuple((vertex['x']+self.box.l, vertex['y']+self.box.t) for vertex in text["boundingPoly"]["vertices"]))
+ if text_box.is_within(bound_box):
+ return text_box
+ return None
+
+ def find_all_text(self, text_to_find, texts_all):
+ found_text_coords = []
+ for text in texts_all:
+ if text["description"] == text_to_find:
+ vertices = Box((vertex['x']+self.box.l, vertex['y']+self.box.t) for vertex in text["boundingPoly"]["vertices"])
+ found_text_coords.append(vertices)
+ return found_text_coords
+
+ def find_all_text_in_bounds(self, text_to_find, bound_box, texts_all):
+ found_text_coords = []
+ for text in texts_all:
+ if text["description"] == text_to_find:
+ text_box = Box((vertex['x']+self.box.l, vertex['y']+self.box.t) for vertex in text["boundingPoly"]["vertices"])
+ if text_box.is_within(bound_box):
+ found_text_coords.append(text_box)
+ return found_text_coords
+
+ def find_texts(self, texts_to_find:list, texts_all):
+ found_texts = {}
+ for text in texts_all:
+ if text["description"] in texts_to_find:
+ vertices = Box(tuple((vertex['x']+self.box.l, vertex['y']+self.box.t) for vertex in text["boundingPoly"]["vertices"]))
+ if not found_texts.get(text["description"]):
+ found_texts[text["description"]] = [vertices]
+ else:
+ found_texts[text["description"]].append(vertices)
+ # found_texts[text["description"]] = vertices
+ return found_texts
+
+ # def get_lines(self):
+ # lines_found = get_image_lines(self.ss)
+
+ # # lines_found[0] += self.box.t
+ # # lines_found[1] += self.box.l
+ # return (lines_found[0] + self.box.t, lines_found[1] + self.box.l)
+
+# Form class definition
+class Form:
+ def __init__(self, heading:str, heading_box:Box, border_box:Box) -> None:
+ self.heading = heading
+ self.heading_box = heading_box
+ self.border_box = border_box
+
+ self.buttons = []
+ self.selected = []
+ self.button_path = "button.png"
+ self.selected_button_path = "button_selected.png"
+ self.square_path = "square.png"
+ self.square_selected_path = "square_selected.png"
+
+ self.ss = None
+ self.ss = ScreenShot(self.border_box.box_tuple, f"form_{self.heading}.png")
+ # self.selected_button_path =
+
+ def establish_borders(self, screen_ss:ScreenShot):
+ # Find bottom border
+ for y in range(self.heading_box.t + self.heading_box.h, screen_ss.box.t + screen_ss.box.h):
+ if screen_ss.is_line_pixel(self.heading_box.l + 1, y, 'horizontal'):
+ self.border_box.h = y - self.heading_box.t
+ break
+
+ # Find right border
+ for x in range(self.heading_box.l + self.heading_box.w, screen_ss.box.l + screen_ss.box.w):
+ if screen_ss.is_line_pixel(x, self.heading_box.t, 'vertical'):
+ self.border_box.w = x - self.heading_box.l
+ break
+ new_box = Box((self.border_box.l, self.border_box.t, self.border_box.w, self.border_box.h))
+ self.border_box = new_box
+ self.ss = ScreenShot(self.border_box.box_tuple, f"form_{self.heading}.png")
+
+ def find_buttons(self, confidence):
+
+ try:
+ self.buttons += self.ss.find_all_img(self.button_path, confidence)
+ except:
+ pass
+
+ try:
+ selected_found = self.ss.find_all_img(self.selected_button_path, confidence)
+ self.buttons += selected_found
+ self.selected += selected_found
+ except:
+ pass
+
+ try:
+ self.buttons += self.ss.find_all_img(self.square_path, confidence)
+ except:
+ pass
+
+ try:
+ selected_found = self.ss.find_all_img(self.square_selected_path, confidence)
+ self.buttons += selected_found
+ self.selected += selected_found
+ except:
+ pass
+ self.selected = selected_found[0]
+ # remove buttons which are within pixels of each other
+ if(len(self.buttons) == 0):
+ return self.buttons
+ self.buttons = [self.buttons[0]] + [self.buttons[i] for i in range(1, len(self.buttons)) if not any(abs(self.buttons[i].l - self.buttons[j].l) < 4 and abs(self.buttons[i].t - self.buttons[j].t) < 4 for j in range(i))]
+ if(len(self.selected) >0):
+ self.selected = [self.selected[0]] + [self.selected[i] for i in range(1, len(self.selected)) if not any(abs(self.selected[i].l - self.selected[j].l) < 4 and abs(self.selected[i].t - self.selected[j].t) < 4 for j in range(i))]
+
+ # self.buttons += self.ss.find_all_img("button_selected.png", confidence)
+ self.buttons = sort_boxes(self.buttons)
+ return self.buttons
+
+ def click_button(self, button_index):
+ try:
+ moveTo(self.buttons[button_index].centre)
+ click()
+ moveTo(100, 400)
+ except:
+ pass
+ def click_last(self):
+ moveTo(self.buttons[-1].centre)
+ click()
+ moveTo(100, 400)
+ def unselect_button(self):
+ for button in self.selected:
+ moveTo(button.centre)
+ click()
+
+ moveTo(100, 400)
+ def click_all_buttons(self):
+ for button in self.buttons:
+ moveTo(button.centre)
+ # click()
+ # time.sleep(1)
+
+# Helper functions
+def pix_within_threshold(pix1, pix2, threshold):
+ return all(abs(pix1[i] - pix2[i]) < threshold for i in range(3))
+
+def find_all_img_in_bounds(image, bound_box, confidence):
+ ss_in_bounds = ScreenShot(bound_box.box_tuple, "temp_ss.png")
+ found_coords = ss_in_bounds.find_all_img(image, confidence)
+ return found_coords
+
+def sort_boxes(boxes):
+ return sorted(boxes, key=lambda box: (box.l,box.t))
+
+def num_within_threshold(num, target, threshold):
+ return abs(num - target) < threshold
+
+
+def get_legit_headings(found, screen_ss):
+ legit_found = {}
+ for heading, verts in found.items():
+ for box in verts:
+ if not any(screen_ss.is_line_pixel(box.l, y, 'horizontal') for y in range(box.t, box.t -8, -1)):
+ # print(f"skipping {heading} because of {box}")
+ continue
+ legit_found[heading] = box
+ break
+ return legit_found
+
+def get_annotations(ss:ScreenShot):
+ server_url = 'https://googlecloudapi.vercel.app/api/v1'
+ with open(ss.path, 'rb') as image:
+ response = post(f'{server_url}/image/annotations', files={'file': image})
+ return response.json()
+
+def num_within_threshold(num, target, threshold):
+ return abs(num - target) < threshold
+
+@eel.expose
+def run_main_code(form_data):
+ screen_size = size()
+ top_bar = 120
+ bottom_bar = 30
+
+
+ # lines = screen_ss.get_lines()
+ # print(lines)
+ # horizontals, verticals = lines
+
+ headings_options_count = {
+ "M0069" : 2,
+ "A1110B": 3,
+
+ "M0150": 13,
+ "M0080": 4,
+ "M0100": 8,
+ "M0110":4,
+
+ "A1250":5,
+
+ "HCS_0110_Facility": 2,
+ # "Blood Pressure": 2,
+ "B0200":5,
+ "B1000":6,
+ "B1300":7,
+
+ "C0100":3,
+ "C0200":4,
+ "C0300A":5,
+ "C0300B":4,
+ "C0300C":4,
+ "C0400A":4,
+ "C0400B":4,
+ "C0400C":4,
+
+ "C1310A":3,
+ "C1310B":4,
+ "C1310C":4,
+ "C1310D":4,
+
+ "M1700":5,
+ "M1710":6,
+ "M1720":5,
+
+ "D0150A1":4,
+ "D0150A2":4,
+
+ "D0150B1":4,
+ "D0150B2":4,
+
+ "D0150C1":4,
+ "D0150C2":4,
+
+ "D0150D1":4,
+ "D0150D2":4,
+
+ "D0150E1":4,
+ "D0150E2":4,
+
+ "D0150F1":4,
+ "D0150F2":4,
+
+ "D0150G1":4,
+ "D0150G2":4,
+
+ "D0150H1":4,
+ "D0150H2":4,
+
+ "D0150I1":4,
+ "D0150I2":4,
+
+ "D0700":7,
+
+ "M1740":3,
+ "M1745":6,
+
+ "M2102_CARE_TYPE_SRC_SPRVSN":5,
+
+ "M1800":4,
+ "M1810":4,
+ "M1820":4,
+ "M1830":7,
+ "M1840":5,
+ "M1845":4,
+ "M1850":6,
+ "M1860":7,
+
+ "GG0130A":22,
+ "GG0130B":22,
+ "GG0130C":22,
+ "GG0130E":22,
+ "GG0130F":22,
+ "GG0130G":22,
+ "GG0130H":22,
+
+ "GG0170A1":11,
+ "GG0170A2":11,
+ "GG0170B1":11,
+ "GG0170B2":11,
+ "GG0170C_MOBILITY_SOCROC_PERF":11,
+ "GG0170C_MOBILITY_DSCHG_GOAL":11,
+ "GG0170D1":11,
+ "GG0170D2":11,
+ "GG0170E1":11,
+ "GG0170E2":11,
+ "GG0170F1":11,
+ "GG0170F2":11,
+ "GG0170G1":11,
+ "GG0170G2":11,
+ "GG0170I1":11,
+ "GG0170I2":11,
+ "GG0170J1":11,
+ "GG0170J2":11,
+ "GG0170K1":11,
+ "GG0170K2":11,
+ "GG0170L1":11,
+ "GG0170L2":11,
+ "GG0170M1":11,
+ "GG0170M2":11,
+ # "GG0170N1":11,
+ "GG0170N2":11,
+ # "GG0170O1":11,
+ "GG0170O2":11,
+ "GG0170P1":11,
+ "GG0170P2":11,
+ "GG0170Q1":11,
+
+ "M1600":4,
+ "M1610":3,
+ "M1620":8,
+ "M1630":3,
+
+ "M1400":5,
+
+ "M1306":2,
+
+ "M1322":5,
+ "M1324":5,
+ "M1330":4,
+ "M1332":4,
+ "M1334":3,
+ "M1340":3,
+ "M1342":4,
+
+ "M1028":9,
+
+ "M1100":12,
+ "M1400":5,
+
+ "M1610":3,
+ "M1620":8,
+ "M1630":3,
+
+ "J0510":6,
+ "J0520":6,
+ "J0530":5,
+
+ "K0520A1":3,
+ "K0520B1":3,
+ "K0520C1":3,
+ "K0520D1":3,
+ "K0520Z1":3,
+
+ "M1870":6,
+
+ "N0415A1":3,
+ "N0415A2":3,
+ "N0415E1":3,
+ "N0415E2":3,
+ "N0415F1":3,
+ "N0415F2":3,
+ "N0415H1":3,
+ "N0415H2":3,
+ "N0415I1":3,
+ "N0415I2":3,
+ "N0415J1":3,
+ "N0415J2":3,
+ "N0415Z1":3,
+
+ "M2001":3,
+ "M2003":2,
+ "M2010":3,
+ "M2020":5,
+ "M2030":6,
+
+ "00110A1a":3,
+ "00110A2a":3,
+ "00110A3a":3,
+ "00110A10a":3,
+ "00110B1a":3,
+ "00110C1a":3,
+ "00110C2a":3,
+ "00110C3a":3,
+ "00110C4a":3,
+ "00110D1a":3,
+ "00110D2a":3,
+ "00110D3a":3,
+ "00110E1a":3,
+ "00110F1a":3,
+ "00110G1a":3,
+ "00110G2a":3,
+ "00110G3a":3,
+ "00110H1a":3,
+ "00110H2a":3,
+ "00110H3a":3,
+ "00110H4a":3,
+ "00110H10a":3,
+ "00110I1a":3,
+ "00110J1a":3,
+ "00110J2a":3,
+ "00110J3a":3,
+
+ "0011001a":3,
+ "0011002a":3,
+ "0011003a":3,
+ "0011004a":3,
+ "00110Z1a":3,
+ }
+ headings_choices = { data["name"]:data["value"] for data in form_data}
+ print(headings_choices)
+ # debugging stuff:
+ found_text = set()
+ found_legit = set()
+ incorrect_buttons = set()
+
+ problem_paths = ["problem_1.png", "problem_2.png"]
+ remaining_headings = list(headings_choices.keys())
+ while len(remaining_headings) > 0:
+ screen_ss = ScreenShot((0, top_bar, screen_size[0], screen_size[1]-top_bar-bottom_bar), "first_ss.png")
+ problems = []
+ for path in problem_paths:
+
+ problems += screen_ss.find_all_img(path, 0.9)
+ for problem in problems:
+ moveTo(problem.centre)
+ click()
+ time.sleep(1)
+ moveTo(200, screen_size[1]/2)
+
+ annotations = get_annotations(screen_ss)["data"]["textAnnotations"]
+ end = screen_ss.find_text("Rehab", annotations)
+ end_2 = screen_ss.find_all_img("end.png", 0.8)
+ if(end):
+
+ print("End found", end_2, end)
+ break
+
+ found = screen_ss.find_texts(headings_choices.keys(), annotations)
+ (found_text.add(head) for head in found.keys())
+ found = get_legit_headings(found, screen_ss)
+ (found_legit.add(head) for head in found.keys())
+
+
+ # headings_boxes = found
+ # print("heading text boxes:", found)
+ # screen_ss = ScreenShot((0, 120, screen_size[0], screen_size[1]-120-40), "first_ss.png")
+ # populate forms on screen:
+ for heading, box in found.items():
+ if(heading not in remaining_headings):
+ continue
+ box = found[heading]
+
+ form_box = Box((
+ box.l,
+ box.t,
+ int(abs(screen_size[0] - box.l)),
+ int(abs(screen_size[1] - box.t))
+ ))
+
+ form_obj = Form(heading, box, form_box)
+ form_obj.establish_borders(screen_ss)
+ form_obj.find_buttons(0.80)
+ if(len(form_obj.buttons) != headings_options_count[heading]):
+ incorrect_buttons.add(heading)
+ print(f"Incorrect buttons for {heading}: {len(form_obj.buttons)}")
+ continue
+ incorrect_buttons.discard(heading)
+
+ form_obj.unselect_button()
+ time.sleep(1)
+ form_obj.find_buttons(0.80)
+ pos_array = []
+ if isinstance(headings_choices[heading], list):
+ pos_array = headings_choices[heading]
+ else:
+ pos_array = [headings_choices[heading]]
+
+ for pos in pos_array:
+ form_obj.click_button(pos)
+ # time.sleep(1)
+ # form_obj.click_button(int(headings_choices[heading]))
+ moveTo(200, screen_size[1]/2)
+ # print(form_obj.buttons)
+ # moveTo((right_border, bottom_border))
+ # time.sleep(2)
+ remaining_headings.remove(heading)
+ pass
+
+ scroll(-100)
+
+
+ never_found = set(headings_choices.keys()) - found_text
+ never_legit = set(headings_choices.keys()) - found_legit
+
+ # write to file:
+ output_file = open("output.txt", "w")
+ print(f"Found: {list(found_text)}")
+ print(f"Never found: {never_found}")
+ output_file.write(f"Found: {list(found_text)}\n")
+ # print(f"Found legit: {found_legit}")
+ print(f"Never legit: {never_legit}")
+ output_file.write(f"Never found: {never_found}\n")
+ output_file.close()
+ print(f"Incorrect buttons: {list(incorrect_buttons)}")
+ block = input("Press enter to continue")
+ pass
+
+if __name__ == '__main__':
+ eel.start('index.html', size=(600, 500))
\ No newline at end of file
diff --git a/ImageProcessing.py b/ImageProcessing.py
new file mode 100644
index 0000000..141317d
--- /dev/null
+++ b/ImageProcessing.py
@@ -0,0 +1,89 @@
+import cv2
+import numpy as np
+from pyautogui import screenshot
+from PIL import Image
+
+
+def image_skeleton(image):
+ size = np.size(image)
+ skel = np.zeros(image.shape, np.uint8)
+
+ ret, img = cv2.threshold(image, 127, 255, 0)
+ element = cv2.getStructuringElement(cv2.MORPH_CROSS, (3, 3))
+ done = False
+ while not done:
+ eroded = cv2.erode(img, element)
+ temp = cv2.dilate(eroded, element)
+ temp = cv2.subtract(img, temp)
+ skel = cv2.bitwise_or(skel, temp)
+ img = eroded.copy()
+
+ zeros = size - cv2.countNonZero(img)
+ if zeros == size:
+ done = True
+ return skel
+
+def get_hough_lines(image):
+ edges = cv2.Canny(image, 50, 250, apertureSize=7)
+ edges = cv2.dilate(edges, np.ones((3, 3), np.uint8))
+ # edges = cv2.erode(edges, np.ones((3, 3), np.uint8))
+ # edges = cv2.dilate(edges, np.ones((3, 3), np.uint8))
+ edges = image_skeleton(edges)
+ # cv2.imshow("Edges", edges)
+ lines = cv2.HoughLines(edges, 1, np.pi/180, 280)
+ lines = lines.reshape(-1, 2)
+ # print(lines)
+ return lines
+
+
+
+def get_cardinal_lines(lines):
+ mask = (lines[:, 1] == 0 ) | (lines[:, 1] == np.pi/2)
+ return lines[mask, :]
+
+def line_coords(lines):
+ # print(lines)
+ horizontal = lines[lines[:, 1] == np.pi/2,0]
+ vertical = lines[lines[:, 1] == 0,0]
+
+ return horizontal, vertical
+
+def draw_lines(image, lines):
+
+ for rho, theta in lines:
+
+ a = np.cos(theta)
+ b = np.sin(theta)
+ x0 = a*rho
+ y0 = b*rho
+ x1 = int(x0 + 1000*(-b))
+ y1 = int(y0 + 1000*(a))
+ x2 = int(x0 - 1000*(-b))
+ y2 = int(y0 - 1000*(a))
+
+ cv2.line(image, (x1, y1), (x2, y2), (0, 0, 255), 2)
+ return image
+
+def get_image_lines(image):
+ if isinstance(image, Image.Image):
+ print(type(image))
+
+ image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+ lines = get_hough_lines(image)
+ # cv2.imshow("Lines", draw_lines(image, lines))
+ cv2.waitKey(0)
+ cv2.destroyAllWindows()
+ lines = line_coords(lines)
+ return lines
+
+if __name__ == "__main__":
+ image = cv2.imread("first_ss.png", 0)
+
+ lines = get_hough_lines(image)
+ cv2.imshow("Lines", draw_lines(cv2.cvtColor(image, cv2.COLOR_GRAY2BGR), lines))
+ lines = get_cardinal_lines(lines)
+ cv2.imshow("Lines 2", draw_lines(cv2.cvtColor(image, cv2.COLOR_GRAY2BGR), lines))
+ print(line_coords(lines))
+ # form_lines = get_cardinal_lines(lines)
+ cv2.waitKey(0)
+ cv2.destroyAllWindows()
\ No newline at end of file
diff --git a/button.png b/button.png
new file mode 100644
index 0000000..eb33330
Binary files /dev/null and b/button.png differ
diff --git a/button_selected.png b/button_selected.png
new file mode 100644
index 0000000..9e11f8e
Binary files /dev/null and b/button_selected.png differ
diff --git a/cloud_api.py b/cloud_api.py
new file mode 100644
index 0000000..7652be5
--- /dev/null
+++ b/cloud_api.py
@@ -0,0 +1,34 @@
+from google.cloud import vision
+
+def detect_text(path):
+ """Detects text in the file."""
+
+ client = vision.ImageAnnotatorClient()
+
+ with open(path, "rb") as image_file:
+ content = image_file.read()
+
+ image = vision.Image(content=content)
+
+ response = client.text_detection(image=image)
+ texts = response.text_annotations
+ print("Texts:")
+
+ for text in texts:
+ print(f'\n"{text.description}"')
+
+ vertices = [
+ f"({vertex.x},{vertex.y})" for vertex in text.bounding_poly.vertices
+ ]
+
+ print("bounds: {}".format(",".join(vertices)))
+
+ if response.error.message:
+ raise Exception(
+ "{}\nFor more info on error messages, check: "
+ "https://cloud.google.com/apis/design/errors".format(response.error.message)
+ )
+
+
+if __name__ == "__main__":
+ detect_text("page.png")
\ No newline at end of file
diff --git a/dist/FormAutomation_2.exe b/dist/FormAutomation_2.exe
new file mode 100644
index 0000000..8e485e4
Binary files /dev/null and b/dist/FormAutomation_2.exe differ
diff --git a/dist/button.png b/dist/button.png
new file mode 100644
index 0000000..457b17f
Binary files /dev/null and b/dist/button.png differ
diff --git a/dist/button_2.png b/dist/button_2.png
new file mode 100644
index 0000000..ec26a31
Binary files /dev/null and b/dist/button_2.png differ
diff --git a/dist/button_selected.png b/dist/button_selected.png
new file mode 100644
index 0000000..7bb3571
Binary files /dev/null and b/dist/button_selected.png differ
diff --git a/dist/problem_1.png b/dist/problem_1.png
new file mode 100644
index 0000000..3ada994
Binary files /dev/null and b/dist/problem_1.png differ
diff --git a/dist/problem_2.png b/dist/problem_2.png
new file mode 100644
index 0000000..cfb1feb
Binary files /dev/null and b/dist/problem_2.png differ
diff --git a/findScreenCoords.py b/findScreenCoords.py
new file mode 100644
index 0000000..e1de2ac
--- /dev/null
+++ b/findScreenCoords.py
@@ -0,0 +1,2 @@
+import pyautogui as pag
+pag.screenshot(region=(360, 611, 17, 15), imageFilename="button.png")
\ No newline at end of file
diff --git a/first_ss.png b/first_ss.png
new file mode 100644
index 0000000..71b8964
Binary files /dev/null and b/first_ss.png differ
diff --git a/form_M0100.png b/form_M0100.png
new file mode 100644
index 0000000..73d1e74
Binary files /dev/null and b/form_M0100.png differ
diff --git a/form_M0150.png b/form_M0150.png
new file mode 100644
index 0000000..c57bb2e
Binary files /dev/null and b/form_M0150.png differ
diff --git a/form_M1800.png b/form_M1800.png
new file mode 100644
index 0000000..8db6ba1
Binary files /dev/null and b/form_M1800.png differ
diff --git a/form_M1810.png b/form_M1810.png
new file mode 100644
index 0000000..4df0c5c
Binary files /dev/null and b/form_M1810.png differ
diff --git a/func.py b/func.py
index 44d8b0e..c21d0e9 100644
--- a/func.py
+++ b/func.py
@@ -1,45 +1,88 @@
-import pyautogui
-import pytesseract
+import pyautogui as pag
from PIL import Image
+import pytesseract
import time
+import webbrowser
+import numpy as np
-# Set up Tesseract executable path if needed
-pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe' # Adjust the path as needed
+# pag.FAILSAFE = False
-def find_and_click_radio_button(button_text):
- # Take a screenshot of the current screen
- screenshot = pyautogui.screenshot()
- screenshot.save("screenshot.png")
+pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
- # Use OCR to extract text from the screenshot
- image = Image.open("screenshot.png")
- text = pytesseract.image_to_string(image)
+def extract_text_from_image(head_region):
+ screenshot = pag.screenshot(region=head_region)
+ print("screenshot taken")
+ text = pytesseract.image_to_string(screenshot)
+ print("text extracted")
+ return text.strip()
- # Find the position of the radio button based on the provided text
- location = pyautogui.locateCenterOnScreen('radio_buttons/' + button_text + '.png')
-
- # If the radio button is found, click on it
- if location:
- pyautogui.click(location)
- return f'Clicked on the radio button for {button_text}'
- else:
- return f'Radio button for {button_text} not found on the screen'
-
-# Example usage
-if __name__ == "__main__":
+def oldMain():
# Open the HTML file in a web browser
- import webbrowser
- webbrowser.open('file://' + 'index.html')
-
- # Give some time to switch to the browser window
- print('You have 5 seconds to switch to the browser window...')
- time.sleep(5)
+ webbrowser.open('index.html')
+ time.sleep(1)
+
+ form_headers = []
+ max_iterations = 25 # Set the maximum number of iterations
+ iteration = 0 # Initialize the iteration counter
+ seen_texts = set() # Initialize a set to store seen texts
+
+ while iteration < max_iterations:
+ time.sleep(1)
+ header = pag.locateOnScreen('form-head.png', confidence=0.8)
+ print(header[0])
+ print(header)
+ if header is None:
+ break
+ # left=header[0]
+ # top=header[1]
+ # width=header[2]
+ # height=header[3]
+ # header_region = (8, 196,61,25)
+ header_region = tuple(int(head) for head in header)
+ print(header_region)
+ header_text = extract_text_from_image(header_region)
+ if header_text in seen_texts:
+ pag.scroll(-100)
+ continue
+
+ form_headers.append(header)
+ seen_texts.add(header_text)
+ iteration += 1
+ pag.scroll(-100)
+
+ radio_buttons = list(pag.locateAllOnScreen('button.png', confidence=0.8))
+
+ num_forms = 2
+ buttons_per_form = 6
+ print(f'buttons found {len(radio_buttons)}\n')
+ print(f'forms found {len(form_headers)}\n')
- # Example data (replace with actual text you are looking for)
- radio_button_texts = ['option1', 'option2', 'option3']
- # Loop through the radio button texts and click them
- for text in radio_button_texts:
- result = find_and_click_radio_button(text)
- print(result)
- time.sleep(1) # Wait for a short duration before the next click
+if __name__ == "__main__":
+ webbrowser.open('index.html')
+ time.sleep(2)
+ ss=pag.screenshot("ss.png")
+ res=pytesseract.image_to_data(ss)
+ print(res)
+
+
+ # form_buttons = []
+ # for i in range(num_forms):
+ # header = form_headers[i]
+ # buttons = radio_buttons[i*buttons_per_form:(i+1)*buttons_per_form]
+ # form_buttons.append((header, buttons))
+
+ # print(form_buttons[0])
+
+ # # Choose which button to click from each form:
+ # click_options = [1, 2]
+
+ # for i, form in enumerate(form_buttons):
+ # button_number = click_options[i]
+ # print(form)
+ # click_location = pag.center(form[1][button_number-1])
+ # pag.click(click_location)
+
+ # time.sleep(2)
+ # submit_button = pag.locateCenterOnScreen('submit.png', confidence=0.8)
+ # pag.click(submit_button)
diff --git a/funcy.py b/funcy.py
new file mode 100644
index 0000000..7916cb5
--- /dev/null
+++ b/funcy.py
@@ -0,0 +1,54 @@
+import pyautogui as pag
+from PIL import Image
+import pytesseract
+import time
+import webbrowser
+
+def capture_entire_page():
+ # Open the HTML file in a web browser
+ webbrowser.open('index.html')
+ time.sleep(2) # Give time for the page to load
+
+ # Initialize variables
+ full_page_image = Image.new('RGB', (0, 0)) # Create an empty image to combine screenshots
+ scroll_height = pag.size()[1] # Get the height of the screen
+ current_scroll = 0
+
+ while True:
+ # Take a screenshot of the visible part of the page
+ ss = pag.screenshot()
+ # If it's the first screenshot, initialize the full page image
+ if full_page_image.size == (0, 0):
+ full_page_image = Image.new('RGB', (ss.width, ss.height * 10))
+
+ # Paste the screenshot onto the full page image
+ full_page_image.paste(ss, (0, current_scroll))
+ current_scroll += ss.height
+
+ # Scroll down
+ pag.scroll(-scroll_height)
+ time.sleep(1)
+ print(f"Captured {current_scroll} pixels of the page.")
+
+ # Check if we have scrolled to the bottom of the page
+ try:
+ if pag.locateOnScreen('submit.png', confidence=0.8):
+ break
+ except pag.ImageNotFoundException:
+ # Handle case where bottom_marker.png is not found
+ print("Bottom marker not found; continuing to scroll.")
+
+ # Save the full page image
+ full_page_image.save('full_page_screenshot.png')
+
+def ocr_screenshot(file_path):
+ # Open the image file
+ img = Image.open(file_path)
+
+ # Perform OCR on the image
+ text = pytesseract.image_to_string(img)
+ print(text)
+
+if __name__ == "__main__":
+ capture_entire_page()
+ ocr_screenshot('full_page_screenshot.png')
diff --git a/heading_options_count.json b/heading_options_count.json
new file mode 100644
index 0000000..ea34899
--- /dev/null
+++ b/heading_options_count.json
@@ -0,0 +1,651 @@
+{
+ "M0069" : {
+ "options":2,
+ "default":null
+ },
+ "A1110B": {
+ "options":3
+ },
+
+ "M0080": {
+ "options":4,
+ "default":null
+ },
+ "M0100": {
+ "options":8,
+ "default":null
+ },
+ "M0110":{
+ "options":4,
+ "default":null
+ },
+
+ "HCS_0110_Facility": {
+ "options":2,
+ "default":null
+ },
+ "B0200":{
+ "options":5,
+ "default":null
+ },
+ "B1000":{
+ "options":6,
+ "default":null
+ },
+ "B1300":{
+ "options":7,
+ "default":null
+ },
+
+ "C0100":{
+ "options":3,
+ "default":null
+ },
+ "C0200":{
+ "options":4,
+ "default":null
+ },
+ "C0300A":{
+ "options":5,
+ "default":null
+ },
+ "C0300B":{
+ "options":4,
+ "default":null
+ },
+ "C0300C":{
+ "options":4,
+ "default":null
+ },
+ "C0400A":{
+ "options":4,
+ "default":null
+ },
+ "C0400B":{
+ "options":4,
+ "default":null
+ },
+ "C0400C":{
+ "options":4,
+ "default":null
+ },
+
+ "C1310A":{
+ "options":3,
+ "default":null
+ },
+ "C1310B":{
+ "options":4,
+ "default":null
+ },
+ "C1310C":{
+ "options":4,
+ "default":null
+ },
+ "C1310D":{
+ "options":4,
+ "default":null
+ },
+
+ "M1700":{
+ "options":5,
+ "default":null
+ },
+ "M1710":{
+ "options":6,
+ "default":null
+ },
+ "M1720":{
+ "options":5,
+ "default":null
+ },
+
+ "D0150A1":{
+ "options":4,
+ "default":null
+ },
+ "D0150A2":{
+ "options":4,
+ "default":null
+ },
+
+ "D0150B1":{
+ "options":4,
+ "default":null
+ },
+ "D0150B2":{
+ "options":4,
+ "default":null
+ },
+
+ "D0150C1":{
+ "options":4,
+ "default":null
+ },
+ "D0150C2":{
+ "options":4,
+ "default":null
+ },
+
+ "D0150D1":{
+ "options":4,
+ "default":null
+ },
+ "D0150D2":{
+ "options":4,
+ "default":null
+ },
+
+ "D0150E1":{
+ "options":4,
+ "default":null
+ },
+ "D0150E2":{
+ "options":4,
+ "default":null
+ },
+
+ "D0150F1":{
+ "options":4,
+ "default":null
+ },
+ "D0150F2":{
+ "options":4,
+ "default":null
+ },
+
+ "D0150G1":{
+ "options":4,
+ "default":null
+ },
+ "D0150G2":{
+ "options":4,
+ "default":null
+ },
+
+ "D0150H1":{
+ "options":4,
+ "default":null
+ },
+ "D0150H2":{
+ "options":4,
+ "default":null
+ },
+
+ "D0150I1":{
+ "options":4,
+ "default":null
+ },
+ "D0150I2":{
+ "options":4,
+ "default":null
+ },
+ "D0700":{
+ "options":7,
+ "default":null
+ },
+
+ "M1740":{
+ "options":3,
+ "default":null
+ },
+ "M1745":{
+ "options":6,
+ "default":null
+ },
+
+ "M2102_CARE_TYPE_SRC_SPRVSN":{
+ "options":5,
+ "default":null
+ },
+
+ "M1800":{
+ "options":4,
+ "default":null
+ },
+ "M1810":{
+ "options":4,
+ "default":null
+ },
+ "M1820":{
+ "options":4,
+ "default":null
+ },
+ "M1830":{
+ "options":4,
+ "default":null
+ },
+ "M1840":{
+ "options":5,
+ "default":null
+ },
+ "M1845":{
+ "options":4,
+ "default":null
+ },
+ "M1850":{
+ "options":6,
+ "default":null
+ },
+ "M1860":{
+ "options":7,
+ "default":null
+ },
+
+ "GG0170A1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170A2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170B1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170B2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170C_MOBILITY_SOCROC_PERF":{
+ "options":11,
+ "default":null
+ },
+ "GG0170C_MOBILITY_DSCHG_GOAL":{
+ "options":11,
+ "default":null
+ },
+ "GG0170D1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170D2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170E1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170E2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170F1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170F2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170G1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170G2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170I1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170I2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170J1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170J2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170K1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170K2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170L1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170L2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170M1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170M2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170N1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170N2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170O1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170O2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170P1":{
+ "options":11,
+ "default":null
+ },
+ "GG0170P2":{
+ "options":11,
+ "default":null
+ },
+ "GG0170Q1":{
+ "options":11,
+ "default":null
+ },
+
+ "M1600":{
+ "options":4,
+ "default":null
+ },
+ "M1610":{
+ "options":3,
+ "default":null
+ },
+ "M1620":{
+ "options":8,
+ "default":null
+ },
+ "M1630":{
+ "options":3,
+ "default":null
+ },
+
+ "M1400":{
+ "options":5,
+ "default":null
+ },
+
+ "M1306":{
+ "options":2,
+ "default":null
+ },
+
+ "M1322":{
+ "options":5,
+ "default":null
+ },
+ "M1324":{
+ "options":5,
+ "default":null
+ },
+ "M1330":{
+ "options":4,
+ "default":null
+ },
+ "M1332":{
+ "options":4,
+ "default":null
+ },
+ "M1334":{
+ "options":3,
+ "default":null
+ },
+ "M1340":{
+ "options":3,
+ "default":null
+ },
+ "M1342":{
+ "options":4,
+ "default":null
+ },
+
+ "J0510":{
+ "options":6,
+ "default":null
+ },
+ "J0520":{
+ "options":6,
+ "default":null
+ },
+ "J0530":{
+ "options":5,
+ "default":null
+ },
+
+ "K0520A1":{
+ "options":3,
+ "default":null
+ },
+ "K0520B1":{
+ "options":3,
+ "default":null
+ },
+ "K0520C1":{
+ "options":3,
+ "default":null
+ },
+ "K0520D1":{
+ "options":3,
+ "default":null
+ },
+ "K0520Z1":{
+ "options":3,
+ "default":null
+ },
+
+ "M1870":{
+ "options":6,
+ "default":null
+ },
+
+ "N0415A1":{
+ "options":3,
+ "default":null
+ },
+ "N0415A2":{
+ "options":3,
+ "default":null
+ },
+ "N0415E1":{
+ "options":3,
+ "default":null
+ },
+ "N0415E2":{
+ "options":3,
+ "default":null
+ },
+ "N0415F1":{
+ "options":3,
+ "default":null
+ },
+ "N0415F2":{
+ "options":3,
+ "default":null
+ },
+ "N0415H1":{
+ "options":3,
+ "default":null
+ },
+ "N0415H2":{
+ "options":3,
+ "default":null
+ },
+ "N0415I1":{
+ "options":3,
+ "default":null
+ },
+ "N0415I2":{
+ "options":3,
+ "default":null
+ },
+ "N0415J1":{
+ "options":3,
+ "default":null
+ },
+ "N0415J2":{
+ "options":3,
+ "default":null
+ },
+ "N0415Z1":{
+ "options":3,
+ "default":null
+ },
+
+ "M2001":{
+ "options":3,
+ "default":null
+ },
+ "M2003":{
+ "options":2,
+ "default":null
+ },
+ "M2010":{
+ "options":3,
+ "default":null
+ },
+ "M2020":{
+ "options":5,
+ "default":null
+ },
+ "M2030":{
+ "options":5,
+ "default":null
+ },
+
+ "00110A1a":{
+ "options":3,
+ "default":null
+ },
+ "00110A2a":{
+ "options":3,
+ "default":null
+ },
+ "00110A3a":{
+ "options":3,
+ "default":null
+ },
+ "00110A10a":{
+ "options":3,
+ "default":null
+ },
+ "00110B1a":{
+ "options":3,
+ "default":null
+ },
+ "00110C1a":{
+ "options":3,
+ "default":null
+ },
+ "00110C2a":{
+ "options":3,
+ "default":null
+ },
+ "00110C3a":{
+ "options":3,
+ "default":null
+ },
+ "00110C4a":{
+ "options":3,
+ "default":null
+ },
+ "00110D1a":{
+ "options":3,
+ "default":null
+ },
+ "00110D2a":{
+ "options":3,
+ "default":null
+ },
+ "00110D3a":{
+ "options":3,
+ "default":null
+ },
+ "00110E1a":{
+ "options":3,
+ "default":null
+ },
+ "00110F1a":{
+ "options":3,
+ "default":null
+ },
+ "00110G1a":{
+ "options":3,
+ "default":null
+ },
+ "00110G2a":{
+ "options":3,
+ "default":null
+ },
+ "00110G3a":{
+ "options":3,
+ "default":null
+ },
+ "00110H1a":{
+ "options":3,
+ "default":null
+ },
+ "00110H2a":{
+ "options":3,
+ "default":null
+ },
+ "00110H3a":{
+ "options":3,
+ "default":null
+ },
+ "00110H4a":{
+ "options":3,
+ "default":null
+ },
+ "00110H10a":{
+ "options":3,
+ "default":null
+ },
+ "00110I1a":{
+ "options":3,
+ "default":null
+ },
+ "00110J1a":{
+ "options":3,
+ "default":null
+ },
+ "00110J2a":{
+ "options":3,
+ "default":null
+ },
+ "00110J3a":{
+ "options":3,
+ "default":null
+ },
+
+ "0011001a":{
+ "options":3,
+ "default":null
+ },
+ "0011002a":{
+ "options":3,
+ "default":null
+ },
+ "0011003a":{
+ "options":3,
+ "default":null
+ },
+ "0011004a":{
+ "options":3,
+ "default":null
+ },
+ "00110Z1a":3
+}
\ No newline at end of file
diff --git a/headings_default_values.json b/headings_default_values.json
new file mode 100644
index 0000000..93a79a8
--- /dev/null
+++ b/headings_default_values.json
@@ -0,0 +1,4 @@
+{
+ "A1110B":0,
+ ""
+}
\ No newline at end of file
diff --git a/index.html b/index.html
index 366268d..c893d59 100644
--- a/index.html
+++ b/index.html
@@ -6,28 +6,53 @@
Radio Button Test
- Select an Option
-
-
+ Select Options
+
+