From a82bd59dd2365de93a695d2821c67906b42fe1ae Mon Sep 17 00:00:00 2001 From: Sean Chatman <136349053+seanchatmangpt@users.noreply.github.com> Date: Thu, 29 Aug 2024 19:51:28 -0700 Subject: [PATCH] pyautomator starting to get off the ground. --- poetry.lock | 30 +- pyproject.toml | 3 + src/dspygen/__init__.py | 1 + src/dspygen/dspygen_app.py | 88 ++-- src/dspygen/experiments/cal_apps/__init__.py | 6 - .../experiments/cal_apps/calendar_set.py | 40 -- src/dspygen/modules/ask_data_module.py | 66 +++ src/dspygen/modules/ask_df_module.py | 48 ++ .../automated_email_responder_module.py | 104 ++-- src/dspygen/modules/create_row_module.py | 101 ++++ src/dspygen/modules/df_sql_module.py | 2 +- .../dspygen/pyautomator/__init__.py | 0 src/dspygen/pyautomator/base_app.py | 70 +++ src/dspygen/pyautomator/calendar/__init__.py | 0 .../calendar}/calendar_app.py | 0 .../calendar_event_integration_testing.py | 7 +- .../calendar}/calendar_integration_testing.py | 2 +- .../pyautomator/calendar/calendar_set.py | 40 ++ .../calendar}/generated_event.ics | 0 src/dspygen/pyautomator/contacts/__init__.py | 0 src/dspygen/pyautomator/contacts/contact.py | 150 ++++++ .../pyautomator/contacts/contacts_main.py | 143 ++++++ .../contacts/organization_contact.py | 42 ++ .../pyautomator/contacts/person_contact.py | 48 ++ src/dspygen/pyautomator/event_kit/__init__.py | 0 .../event_kit}/alarm.py | 0 .../event_kit}/alarm_integration_testing.py | 5 +- .../event_kit}/calendar_event.py | 0 .../event_kit}/calendar_event_list.py | 3 +- .../event_kit}/calendar_item.py | 0 .../event_kit}/event_store.py | 2 +- .../event_kit}/recurrence_rule.py | 0 .../event_kit}/reminder.py | 5 +- .../event_kit}/reminder_list.py | 3 +- src/dspygen/pyautomator/linkedin/__init__.py | 0 .../pyautomator/linkedin/linkedin_app.py | 78 +++ .../pyautomator/linkedin/linkedin_profile.md | 486 ++++++++++++++++++ .../pyautomator/linkedin/sales_nav_app.py | 142 +++++ src/dspygen/pyautomator/reminders/__init__.py | 0 .../reminders}/reminder_app.py | 34 +- .../reminder_integration_testing.py | 6 +- .../reminders}/wintermute_reminder.py | 3 +- src/dspygen/pyautomator/safari/__init__.py | 0 src/dspygen/pyautomator/safari/safari_app.py | 143 ++++++ src/dspygen/rm/data_retriever.py | 8 + src/dspygen/rm/doc_retriever.py | 7 +- src/dspygen/rm/retrievers.README.md | 159 ++++++ src/dspygen/subcommands/wkf_cmd.py | 287 ++++++++++- .../workflow/data_analysis_workflow.yaml | 6 +- src/dspygen/workflow/workflow_executor.py | 101 +++- src/dspygen/workflow/workflow_models.py | 15 +- src/dspygen/writer/Tetris_Blog_Phi3Med.md | 6 + src/dspygen/writer/data_writer.py | 84 ++- tests/actor/test_actor.py | 66 +-- .../cal_apps/test_reminder_app_bdd.py | 11 +- tests/experiments/test_reminders_models.py | 1 - ...dvanced_integration_job_search_workflow.py | 0 tests/test_create_row_integration.py | 77 +++ tests/test_data_retriever_stress.py | 83 +++ tests/test_event_kit_service.py | 280 +++++----- ...st_integration_email_responder_workflow.py | 122 +++++ ...dvanced_integration_job_search_workflow.py | 181 +++++++ tests/test_wkf_cmd.py | 86 ++++ tests/test_workflow_integration.py | 60 +++ tests/test_workflow_runner.py | 24 + tests/test_workflow_scheduler.py | 115 +++++ tests/test_workflow_unit.py | 91 ++++ 67 files changed, 3350 insertions(+), 421 deletions(-) delete mode 100644 src/dspygen/experiments/cal_apps/__init__.py delete mode 100644 src/dspygen/experiments/cal_apps/calendar_set.py create mode 100644 src/dspygen/modules/ask_data_module.py create mode 100644 src/dspygen/modules/ask_df_module.py create mode 100644 src/dspygen/modules/create_row_module.py rename tests/experiments/cal_apps/test_reminder_cal_item.py => src/dspygen/pyautomator/__init__.py (100%) create mode 100644 src/dspygen/pyautomator/base_app.py create mode 100644 src/dspygen/pyautomator/calendar/__init__.py rename src/dspygen/{experiments/cal_apps => pyautomator/calendar}/calendar_app.py (100%) rename src/dspygen/{experiments/cal_apps => pyautomator/calendar}/calendar_event_integration_testing.py (97%) rename src/dspygen/{experiments/cal_apps => pyautomator/calendar}/calendar_integration_testing.py (97%) create mode 100644 src/dspygen/pyautomator/calendar/calendar_set.py rename src/dspygen/{experiments/cal_apps => pyautomator/calendar}/generated_event.ics (100%) create mode 100644 src/dspygen/pyautomator/contacts/__init__.py create mode 100644 src/dspygen/pyautomator/contacts/contact.py create mode 100644 src/dspygen/pyautomator/contacts/contacts_main.py create mode 100644 src/dspygen/pyautomator/contacts/organization_contact.py create mode 100644 src/dspygen/pyautomator/contacts/person_contact.py create mode 100644 src/dspygen/pyautomator/event_kit/__init__.py rename src/dspygen/{experiments/cal_apps => pyautomator/event_kit}/alarm.py (100%) rename src/dspygen/{experiments/cal_apps => pyautomator/event_kit}/alarm_integration_testing.py (95%) rename src/dspygen/{experiments/cal_apps => pyautomator/event_kit}/calendar_event.py (100%) rename src/dspygen/{experiments/cal_apps => pyautomator/event_kit}/calendar_event_list.py (97%) rename src/dspygen/{experiments/cal_apps => pyautomator/event_kit}/calendar_item.py (100%) rename src/dspygen/{experiments/cal_apps => pyautomator/event_kit}/event_store.py (98%) rename src/dspygen/{experiments/cal_apps => pyautomator/event_kit}/recurrence_rule.py (100%) rename src/dspygen/{experiments/cal_apps => pyautomator/event_kit}/reminder.py (98%) rename src/dspygen/{experiments/cal_apps => pyautomator/event_kit}/reminder_list.py (98%) create mode 100644 src/dspygen/pyautomator/linkedin/__init__.py create mode 100644 src/dspygen/pyautomator/linkedin/linkedin_app.py create mode 100644 src/dspygen/pyautomator/linkedin/linkedin_profile.md create mode 100644 src/dspygen/pyautomator/linkedin/sales_nav_app.py create mode 100644 src/dspygen/pyautomator/reminders/__init__.py rename src/dspygen/{experiments/cal_apps => pyautomator/reminders}/reminder_app.py (94%) rename src/dspygen/{experiments/cal_apps => pyautomator/reminders}/reminder_integration_testing.py (97%) rename src/dspygen/{experiments/cal_apps => pyautomator/reminders}/wintermute_reminder.py (92%) create mode 100644 src/dspygen/pyautomator/safari/__init__.py create mode 100644 src/dspygen/pyautomator/safari/safari_app.py create mode 100644 src/dspygen/rm/retrievers.README.md create mode 100644 src/dspygen/writer/Tetris_Blog_Phi3Med.md delete mode 100644 tests/experiments/test_reminders_models.py rename src/dspygen/experiments/cal_apps/recurrence_handler.py => tests/test_advanced_integration_job_search_workflow.py (100%) create mode 100644 tests/test_create_row_integration.py create mode 100644 tests/test_data_retriever_stress.py create mode 100644 tests/test_integration_email_responder_workflow.py create mode 100644 tests/test_ultra_advanced_integration_job_search_workflow.py create mode 100644 tests/test_wkf_cmd.py create mode 100644 tests/test_workflow_integration.py create mode 100644 tests/test_workflow_runner.py create mode 100644 tests/test_workflow_scheduler.py create mode 100644 tests/test_workflow_unit.py diff --git a/poetry.lock b/poetry.lock index e241e11..8415e0e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -232,6 +232,34 @@ files = [ {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"}, ] +[[package]] +name = "apscheduler" +version = "3.10.4" +description = "In-process task scheduler with Cron-like capabilities" +optional = false +python-versions = ">=3.6" +files = [ + {file = "APScheduler-3.10.4-py3-none-any.whl", hash = "sha256:fb91e8a768632a4756a585f79ec834e0e27aad5860bac7eaa523d9ccefd87661"}, + {file = "APScheduler-3.10.4.tar.gz", hash = "sha256:e6df071b27d9be898e486bc7940a7be50b4af2e9da7c08f0744a96d4bd4cef4a"}, +] + +[package.dependencies] +pytz = "*" +six = ">=1.4.0" +tzlocal = ">=2.0,<3.dev0 || >=4.dev0" + +[package.extras] +doc = ["sphinx", "sphinx-rtd-theme"] +gevent = ["gevent"] +mongodb = ["pymongo (>=3.0)"] +redis = ["redis (>=3.0)"] +rethinkdb = ["rethinkdb (>=2.4.0)"] +sqlalchemy = ["sqlalchemy (>=1.4)"] +testing = ["pytest", "pytest-asyncio", "pytest-cov", "pytest-tornado5"] +tornado = ["tornado (>=4.3)"] +twisted = ["twisted"] +zookeeper = ["kazoo"] + [[package]] name = "argon2-cffi" version = "23.1.0" @@ -11540,4 +11568,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.10,<4.0" -content-hash = "4fa33866b04135213f403d3e605b177ad3ec01dfdc44b7bb5a5434207468fc2b" +content-hash = "e974f6cdc8b5a3b9448f1a58a1d792b8314b9ba164457882626e63422d4fc30e" diff --git a/pyproject.toml b/pyproject.toml index 6d1d5ca..75fd0c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,9 @@ trafilatura = "^1.12.1" pyobjc = "^10.3.1" pyobjc-framework-eventkit = "^10.3.1" icalendar = "^5.0.13" +pyobjc-framework-contacts = "^10.3.1" +pytz = "^2024.1" +apscheduler = "^3.10.4" [tool.poetry.group.test.dependencies] # https://python-poetry.org/docs/master/managing-dependencies/ coverage = { extras = ["toml"], version = ">=7.2.5" } diff --git a/src/dspygen/__init__.py b/src/dspygen/__init__.py index 6725c79..080a23e 100644 --- a/src/dspygen/__init__.py +++ b/src/dspygen/__init__.py @@ -11,4 +11,5 @@ def configure_injector(binder): event_store = EventKit.EKEventStore.alloc().init() binder.bind(EventKit.EKEventStore, event_store) + inject.configure(configure_injector) \ No newline at end of file diff --git a/src/dspygen/dspygen_app.py b/src/dspygen/dspygen_app.py index 89cbcd3..951993c 100644 --- a/src/dspygen/dspygen_app.py +++ b/src/dspygen/dspygen_app.py @@ -1,44 +1,44 @@ -"""Streamlit app.""" - -from importlib.metadata import version - -import streamlit as st - -from dspygen.modules.chat_bot_module import chat_bot_call -from dspygen.modules.insight_tweet_module import insight_tweet_call -from dspygen.modules.streamlit_bot_module import streamlit_bot_call -from dspygen.utils.dspy_tools import init_dspy -from dspygen.utils.file_tools import source_dir, pages_dir - -st.title(f"dspygen v{version('dspygen')}") # type: ignore[no-untyped-call] - - -# # Streamlit form and display -# st.title("Insight Tweet Generator") -# -# insight_input = st.text_input("Enter your insight:") -# -# if st.button("Generate Tweet"): -# init_dspy() -# result = insight_tweet_call(insight_input) -# st.write(result) - -from st_pages import Page, show_pages, add_page_title - -# Optional -- adds the title and icon to the current page -add_page_title() - -# Specify what pages should be shown in the sidebar, and what their titles and icons -# should be - -page_list = [Page(str(source_dir("app.py")), "Home", "🏠")] - -# loop through the pages and display them -for page_src in pages_dir().iterdir(): - if page_src.is_file() and page_src.suffix == ".py": - page_list.append(Page(str(page_src), page_src.stem, ":books:")) - -# Remove __init__.py from the list -page_list = [page for page in page_list if page.name != "init"] - -# show_pages(page_list) +# """Streamlit app.""" +# +# from importlib.metadata import version +# +# import streamlit as st +# +# from dspygen.modules.chat_bot_module import chat_bot_call +# from dspygen.modules.insight_tweet_module import insight_tweet_call +# from dspygen.modules.streamlit_bot_module import streamlit_bot_call +# from dspygen.utils.dspy_tools import init_dspy +# from dspygen.utils.file_tools import source_dir, pages_dir +# +# st.title(f"dspygen v{version('dspygen')}") # type: ignore[no-untyped-call] +# +# +# # # Streamlit form and display +# # st.title("Insight Tweet Generator") +# # +# # insight_input = st.text_input("Enter your insight:") +# # +# # if st.button("Generate Tweet"): +# # init_dspy() +# # result = insight_tweet_call(insight_input) +# # st.write(result) +# +# from st_pages import Page, show_pages, add_page_title +# +# # Optional -- adds the title and icon to the current page +# add_page_title() +# +# # Specify what pages should be shown in the sidebar, and what their titles and icons +# # should be +# +# page_list = [Page(str(source_dir("app.py")), "Home", "🏠")] +# +# # loop through the pages and display them +# for page_src in pages_dir().iterdir(): +# if page_src.is_file() and page_src.suffix == ".py": +# page_list.append(Page(str(page_src), page_src.stem, ":books:")) +# +# # Remove __init__.py from the list +# page_list = [page for page in page_list if page.name != "init"] +# +# # show_pages(page_list) diff --git a/src/dspygen/experiments/cal_apps/__init__.py b/src/dspygen/experiments/cal_apps/__init__.py deleted file mode 100644 index 354e4b0..0000000 --- a/src/dspygen/experiments/cal_apps/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .calendar_event import CalendarEvent -from .alarm import Alarm -from .recurrence_rule import RecurrenceRule -from .calendar_item import CalendarItem - -__all__ = ['CalendarEvent', 'Alarm', 'RecurrenceRule', 'CalendarItem'] diff --git a/src/dspygen/experiments/cal_apps/calendar_set.py b/src/dspygen/experiments/cal_apps/calendar_set.py deleted file mode 100644 index ab5dcdf..0000000 --- a/src/dspygen/experiments/cal_apps/calendar_set.py +++ /dev/null @@ -1,40 +0,0 @@ -import objc -import EventKit -from Foundation import NSDateComponents, NSCalendar - -# Initialize the Event Store -event_store = EventKit.EKEventStore.alloc().init() - -# Request access to reminders -def request_access_callback(granted, error): - if not granted: - raise PermissionError("Access to reminders denied.") - -event_store.requestAccessToEntityType_completion_(EventKit.EKEntityTypeReminder, request_access_callback) - -# Get the default calendar for new reminders -default_calendar = event_store.defaultCalendarForNewReminders() - -# Create a new reminder -reminder = EventKit.EKReminder.reminderWithEventStore_(event_store) - -# Set the reminder properties using the appropriate setter methods or properties -reminder.setTitle_("New Reminder") -reminder.setNotes_("This is a test reminder.") -reminder.setCalendar_(default_calendar) # Set the calendar - -# Set the due date using NSDateComponents -due_date = NSDateComponents.alloc().init() -due_date.setYear_(2024) -due_date.setMonth_(8) -due_date.setDay_(28) -due_date.setHour_(7) - -reminder.setDueDateComponents_(due_date) - -# Save the reminder -success, error = event_store.saveReminder_commit_error_(reminder, True, objc.nil) -if not success: - raise Exception("Failed to save reminder:", error) - -print(f"Reminder '{reminder.title()}' set in calendar '{reminder.calendar().title()}'") \ No newline at end of file diff --git a/src/dspygen/modules/ask_data_module.py b/src/dspygen/modules/ask_data_module.py new file mode 100644 index 0000000..7ebda4c --- /dev/null +++ b/src/dspygen/modules/ask_data_module.py @@ -0,0 +1,66 @@ +import dspy +from dspygen.utils.dspy_tools import init_dspy, init_ol +from dspygen.rm.data_retriever import read_any +from dspygen.rm.doc_retriever import read_any as doc_read_any +import pandas as pd +import io + +class AskDataSignature(dspy.Signature): + """ + Answers a natural language question about data from a file. + """ + question = dspy.InputField(desc="Natural language question about the data.") + data = dspy.InputField(desc="The data content from the file.") + answer = dspy.OutputField(desc="Plain text answer to the question.") + +class AskDataModule(dspy.Module): + """AskDataModule for answering questions about data from various file types""" + + def __init__(self, **forward_args): + super().__init__() + self.forward_args = forward_args + + def forward(self, question, file_path): + try: + # First, try to read as structured data + data = read_any(file_path, query="") + if isinstance(data, pd.DataFrame): + csv_buffer = io.StringIO() + data.to_csv(csv_buffer, index=False) + data = csv_buffer.getvalue() + else: + data = str(data) + except Exception: + try: + # If that fails, try to read as a document + data = doc_read_any(file_path) + if isinstance(data, dict): + data = "\n".join(data.values()) + data = str(data) + except Exception: + # If both fail, read as plain text + with open(file_path, 'r', encoding='utf-8') as file: + data = file.read() + + pred = dspy.Predict(AskDataSignature) + return pred(question=question, data=data).answer + +def ask_data_call(question, file_path): + ask_data_module = AskDataModule() + return ask_data_module.forward(question=question, file_path=file_path) + +def main(): + # init_ol(model="mistral-nemo") + init_ol(model="qwen2:latest") + # init_ol(model="mistral-nemo") + # Example usage + from dspygen.experiments.cal_apps.reminder_app import RemindersApp + app = RemindersApp() + app.export_reminders("reminders.csv") + question = "Can you answer me a new appointment for a haircut at 1pm on 9/1" + + result = ask_data_call(question=question, file_path="reminders.csv") + print(result) + +if __name__ == "__main__": + main() diff --git a/src/dspygen/modules/ask_df_module.py b/src/dspygen/modules/ask_df_module.py new file mode 100644 index 0000000..24572f2 --- /dev/null +++ b/src/dspygen/modules/ask_df_module.py @@ -0,0 +1,48 @@ +import dspy +from dspygen.utils.dspy_tools import init_dspy +import pandas as pd +import io + +class AskDFSignature(dspy.Signature): + """ + Answers a natural language question about a DataFrame. + """ + question = dspy.InputField(desc="Natural language question about the DataFrame.") + df_csv = dspy.InputField(desc="The DataFrame in CSV string format.") + answer = dspy.OutputField(desc="Plain text answer to the question.") + +class AskDFModule(dspy.Module): + """AskDFModule for answering questions about DataFrames using natural language""" + + def __init__(self, **forward_args): + super().__init__() + self.forward_args = forward_args + + def forward(self, question, df): + # Convert DataFrame to CSV string + csv_buffer = io.StringIO() + df.to_csv(csv_buffer, index=False) + df_csv = csv_buffer.getvalue() + + pred = dspy.Predict(AskDFSignature) + return pred(question=question, df_csv=df_csv).answer + +def ask_df_call(question, df): + ask_df_module = AskDFModule() + return ask_df_module.forward(question=question, df=df) + +def main(): + init_dspy() + # Example usage + df = pd.DataFrame({ + 'name': ['Alice', 'Bob', 'Charlie'], + 'age': [25, 30, 35], + 'city': ['New York', 'San Francisco', 'London'] + }) + question = "Who is older than 30?" + + result = ask_df_call(question=question, df=df) + print(result) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/dspygen/modules/automated_email_responder_module.py b/src/dspygen/modules/automated_email_responder_module.py index 9744aa2..1eee39d 100644 --- a/src/dspygen/modules/automated_email_responder_module.py +++ b/src/dspygen/modules/automated_email_responder_module.py @@ -1,91 +1,51 @@ -""" - -""" import dspy -from dspygen.utils.dspy_tools import init_dspy +import pandas as pd +import io +from dspygen.rm.doc_retriever import DocRetriever +from dspygen.utils.dspy_tools import init_dspy + + +class AutomatedEmailResponderSignature(dspy.Signature): + """ + Generates a response to an email considering the LinkedIn profile. + """ + linkedin_profile = dspy.InputField(desc="The LinkedIn profile in text format.") + email_message = dspy.InputField(desc="The incoming email message.") + response = dspy.OutputField(desc="Generated response to the email.") class AutomatedEmailResponderModule(dspy.Module): - """AutomatedEmailResponderModule""" - + """AutomatedEmailResponderModule for responding to emails considering LinkedIn profile""" + def __init__(self, **forward_args): super().__init__() self.forward_args = forward_args - self.output = None - - def __or__(self, other): - if other.output is None and self.output is None: - self.forward(**self.forward_args) - - other.pipe(self.output) - - return other - - def forward(self, email_messages): - pred = dspy.Predict("email_messages -> responses") - self.output = pred(email_messages=email_messages).responses - return self.output - - def pipe(self, input_str): - raise NotImplementedError("Please implement the pipe method for DSL support.") - # Replace TODO with a keyword from you forward method - # return self.forward(TODO=input_str) - - -from typer import Typer -app = Typer() - - -@app.command() -def call(email_messages): - """AutomatedEmailResponderModule""" - init_dspy() - print(automated_email_responder_call(email_messages=email_messages)) + def forward(self, email_message, linkedin_profile): + pred = dspy.ChainOfThought(AutomatedEmailResponderSignature) + return pred(email_message=email_message, linkedin_profile=linkedin_profile).response - -def automated_email_responder_call(email_messages): - automated_email_responder = AutomatedEmailResponderModule() - return automated_email_responder.forward(email_messages=email_messages) - +def automated_email_call(email_message, linkedin_profile): + module = AutomatedEmailResponderModule() + return module.forward(email_message=email_message, linkedin_profile=linkedin_profile) def main(): - init_dspy() - email_messages = "" - result = automated_email_responder_call(email_messages=email_messages) - print(result) - - - -from fastapi import APIRouter -router = APIRouter() - -@router.post("/automated_email_responder/") -async def automated_email_responder_route(data: dict): - # Your code generation logic here - init_dspy() - - print(data) - return automated_email_responder_call(**data) - - - -""" -import streamlit as st + from dspygen.utils.dspy_tools import init_ol, init_dspy + init_ol(model="mistral-nemo") + # init_dspy() + # Retrieve LinkedIn profile + linkedin_profile = DocRetriever("/Users/sac/dev/dspygen/src/dspygen/experiments/pyautomator/linkedin_profile.md").forward() -# Streamlit form and display -st.title("AutomatedEmailResponderModule Generator") -email_messages = st.text_input("Enter email_messages") + # Example email message + email_message = "Hello, I saw your profile and I'm interested in discussing a potential job opportunity. Can we schedule a call?" -if st.button("Submit AutomatedEmailResponderModule"): - init_dspy() + response = automated_email_call(email_message, linkedin_profile) + print("Generated Response:") + print(response) - result = automated_email_responder_call(email_messages=email_messages) - st.write(result) -""" -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/src/dspygen/modules/create_row_module.py b/src/dspygen/modules/create_row_module.py new file mode 100644 index 0000000..793d89c --- /dev/null +++ b/src/dspygen/modules/create_row_module.py @@ -0,0 +1,101 @@ +import dspy +from dspygen.utils.dspy_tools import init_dspy +import logging +import json +import pandas as pd +import numpy as np + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def convert_to_serializable(obj): + if isinstance(obj, (np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64)): + return int(obj) + elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)): + return float(obj) + elif isinstance(obj, np.bool_): + return bool(obj) + elif isinstance(obj, np.ndarray): + return obj.tolist() + return obj + +class CreateRowSignature(dspy.Signature): + """ + Creates a new row for a list of dictionaries based on a natural language request. + """ + data_sample = dspy.InputField(desc="The entire existing dataset to understand the structure.") + schema = dspy.InputField(desc="The schema of the data, including column names and types.") + request = dspy.InputField(desc="Natural language request to add a new row.") + new_row = dspy.OutputField(desc="A JSON string representing the new row to be added.") + +class CreateRowModule(dspy.Module): + """CreateRowModule for adding a new row to a list of dictionaries based on a natural language request""" + + def __init__(self, **forward_args): + super().__init__() + self.forward_args = forward_args + + def forward(self, data, request): + if not data: + raise ValueError("Input data is empty.") + + # Convert list of dictionaries to DataFrame + df = pd.DataFrame(data) + + # Use the entire dataset as a sample + data_sample = df.to_json(orient='records') + + # Create schema information + schema = {col: str(dtype) for col, dtype in df.dtypes.items()} + + pred = dspy.Predict(CreateRowSignature) + new_row_str = pred(data_sample=data_sample, schema=json.dumps(schema), request=request).new_row + + # Parse the new_row string into a dictionary + try: + new_row = json.loads(new_row_str) + except json.JSONDecodeError: + raise ValueError("Failed to parse new_row as JSON. Ensure the model outputs valid JSON.") + + # Ensure all columns from the DataFrame are present in the new row + for col in df.columns: + if col not in new_row: + new_row[col] = None + else: + # Convert the value to the same type as in the DataFrame + new_row[col] = df[col].dtype.type(new_row[col]) + # Convert to serializable type + new_row[col] = convert_to_serializable(new_row[col]) + + # Add the new row to the data + updated_data = data + [new_row] + + return updated_data + +def create_row_call(data, request): + try: + create_row_module = CreateRowModule() + return create_row_module.forward(data=data, request=request) + except Exception as e: + logger.error(f"Error in create_row_call: {e}") + raise + +def main(): + init_dspy() + # Example usage + data = [ + {'name': 'Alice', 'age': 25, 'city': 'New York', 'joined_date': '2023-01-01'}, + {'name': 'Bob', 'age': 30, 'city': 'San Francisco', 'joined_date': '2023-02-01'} + ] + request = "Add a new person named Charlie, who is 35 years old, lives in London, and joined on March 1, 2023" + + try: + result_data = create_row_call(data=data, request=request) + print(json.dumps(result_data, indent=2)) + pd.DataFrame(result_data).to_csv("results.csv", index=False) + except Exception as e: + logger.error(f"An error occurred: {e}") + +if __name__ == "__main__": + main() diff --git a/src/dspygen/modules/df_sql_module.py b/src/dspygen/modules/df_sql_module.py index a75c8e1..4b33f2f 100644 --- a/src/dspygen/modules/df_sql_module.py +++ b/src/dspygen/modules/df_sql_module.py @@ -59,7 +59,7 @@ def dfsql_call(text, df_schema, df_data): def main(): init_dspy() - # app = ReminderApp() + # app = RemindersApp() # app.export_reminders("reminders.csv") # dr = DataRetriever(file_path="reminders.csv") # df_schema = dr.df.columns.tolist() diff --git a/tests/experiments/cal_apps/test_reminder_cal_item.py b/src/dspygen/pyautomator/__init__.py similarity index 100% rename from tests/experiments/cal_apps/test_reminder_cal_item.py rename to src/dspygen/pyautomator/__init__.py diff --git a/src/dspygen/pyautomator/base_app.py b/src/dspygen/pyautomator/base_app.py new file mode 100644 index 0000000..e97504e --- /dev/null +++ b/src/dspygen/pyautomator/base_app.py @@ -0,0 +1,70 @@ +import subprocess +import logging +import os +import tempfile + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +class BaseApp: + def __init__(self, app_name: str, script_dir: str = None): + self.app_name = app_name + self.script_dir = script_dir or tempfile.gettempdir() + + def execute_jxa(self, script: str, save_to_file: bool = False): + if save_to_file: + script_path = self.save_script(script) + logger.debug(f"Script saved to: {script_path}") + else: + script_path = None + + logger.debug(f"Executing JXA script:\n{script}") + + try: + if script_path: + result = subprocess.run(['osascript', '-l', 'JavaScript', script_path], check=True, capture_output=True, + text=True) + else: + result = subprocess.run(['osascript', '-l', 'JavaScript', '-e', script], check=True, + capture_output=True, text=True) + + logger.debug(f"Script result: {result.stdout}") + return result.stdout.strip() + except subprocess.CalledProcessError as e: + logger.error(f"JXA script failed: {e.stderr}") + raise + + def save_script(self, script: str, filename: str = None) -> str: + if not filename: + filename = f"{self.app_name}_script_{int(datetime.now().timestamp())}.jxa" + script_path = os.path.join(self.script_dir, filename) + + with open(script_path, 'w') as file: + file.write(script) + + logger.debug(f"Script written to: {script_path}") + return script_path + + def request_access(self, entity_type: str, save_to_file: bool = False): + script = f""" + const app = Application.currentApplication(); + app.includeStandardAdditions = true; + app.requestAccessToEntityTypeCompletion("{entity_type}", (granted, error) => {{ + if (!granted) {{ + throw new Error('Access to {entity_type} denied.'); + }} + }}); + """ + return self.execute_jxa(script, save_to_file=save_to_file) + + def activate_app(self, save_to_file: bool = False): + script = f""" + const app = Application("{self.app_name}"); + app.activate(); + """ + return self.execute_jxa(script, save_to_file=save_to_file) + + def get_app(self, save_to_file: bool = False): + script = f"Application('{self.app_name}');" + return self.execute_jxa(script, save_to_file=save_to_file) diff --git a/src/dspygen/pyautomator/calendar/__init__.py b/src/dspygen/pyautomator/calendar/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dspygen/experiments/cal_apps/calendar_app.py b/src/dspygen/pyautomator/calendar/calendar_app.py similarity index 100% rename from src/dspygen/experiments/cal_apps/calendar_app.py rename to src/dspygen/pyautomator/calendar/calendar_app.py diff --git a/src/dspygen/experiments/cal_apps/calendar_event_integration_testing.py b/src/dspygen/pyautomator/calendar/calendar_event_integration_testing.py similarity index 97% rename from src/dspygen/experiments/cal_apps/calendar_event_integration_testing.py rename to src/dspygen/pyautomator/calendar/calendar_event_integration_testing.py index 6c51637..44fcff0 100644 --- a/src/dspygen/experiments/cal_apps/calendar_event_integration_testing.py +++ b/src/dspygen/pyautomator/calendar/calendar_event_integration_testing.py @@ -1,12 +1,15 @@ from datetime import datetime, timedelta import inject import EventKit -from dspygen.experiments.cal_apps.calendar_event import CalendarEvent -from dspygen.experiments.cal_apps.alarm import Alarm + import subprocess import tempfile import os +from dspygen.pyautomator.event_kit.alarm import Alarm +from dspygen.pyautomator.event_kit.calendar_event import CalendarEvent + + @inject.autoparams() def create_event(event_store: EventKit.EKEventStore, title: str, calendar: EventKit.EKCalendar, start_date: datetime, end_date: datetime): diff --git a/src/dspygen/experiments/cal_apps/calendar_integration_testing.py b/src/dspygen/pyautomator/calendar/calendar_integration_testing.py similarity index 97% rename from src/dspygen/experiments/cal_apps/calendar_integration_testing.py rename to src/dspygen/pyautomator/calendar/calendar_integration_testing.py index 5f5e145..18dbaa5 100644 --- a/src/dspygen/experiments/cal_apps/calendar_integration_testing.py +++ b/src/dspygen/pyautomator/calendar/calendar_integration_testing.py @@ -1,8 +1,8 @@ from datetime import datetime, timedelta import logging +from dspygen.pyautomator.calendar.calendar_app import CalendarApp from dspygen.utils.dspy_tools import init_dspy -from dspygen.experiments.cal_apps.calendar_app import CalendarApp logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) diff --git a/src/dspygen/pyautomator/calendar/calendar_set.py b/src/dspygen/pyautomator/calendar/calendar_set.py new file mode 100644 index 0000000..98f510c --- /dev/null +++ b/src/dspygen/pyautomator/calendar/calendar_set.py @@ -0,0 +1,40 @@ +# import objc +# import EventKit +# from Foundation import NSDateComponents, NSCalendar +# +# # Initialize the Event Store +# event_store = EventKit.EKEventStore.alloc().init() +# +# # Request access to reminders +# def request_access_callback(granted, error): +# if not granted: +# raise PermissionError("Access to reminders denied.") +# +# event_store.requestAccessToEntityType_completion_(EventKit.EKEntityTypeReminder, request_access_callback) +# +# # Get the default calendar for new reminders +# default_calendar = event_store.defaultCalendarForNewReminders() +# +# # Create a new reminder +# reminder = EventKit.EKReminder.reminderWithEventStore_(event_store) +# +# # Set the reminder properties using the appropriate setter methods or properties +# reminder.setTitle_("New Reminder") +# reminder.setNotes_("This is a test reminder.") +# reminder.setCalendar_(default_calendar) # Set the calendar +# +# # Set the due date using NSDateComponents +# due_date = NSDateComponents.alloc().init() +# due_date.setYear_(2024) +# due_date.setMonth_(8) +# due_date.setDay_(28) +# due_date.setHour_(7) +# +# reminder.setDueDateComponents_(due_date) +# +# # Save the reminder +# success, error = event_store.saveReminder_commit_error_(reminder, True, objc.nil) +# if not success: +# raise Exception("Failed to save reminder:", error) +# +# print(f"Reminder '{reminder.title()}' set in calendar '{reminder.calendar().title()}'") \ No newline at end of file diff --git a/src/dspygen/experiments/cal_apps/generated_event.ics b/src/dspygen/pyautomator/calendar/generated_event.ics similarity index 100% rename from src/dspygen/experiments/cal_apps/generated_event.ics rename to src/dspygen/pyautomator/calendar/generated_event.ics diff --git a/src/dspygen/pyautomator/contacts/__init__.py b/src/dspygen/pyautomator/contacts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dspygen/pyautomator/contacts/contact.py b/src/dspygen/pyautomator/contacts/contact.py new file mode 100644 index 0000000..dd5ec72 --- /dev/null +++ b/src/dspygen/pyautomator/contacts/contact.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import objc +from Contacts import (CNMutableContact, CNContactStore, CNLabeledValue, CNSaveRequest, + CNContact, CNPostalAddress, CNContactFormatter, CNPostalAddressFormatter, + CNLabelHome, CNLabelWork, CNPhoneNumber, CNMutablePostalAddress, CNContactFormatterStyleFullName) +from Foundation import NSString, NSDateComponents +from typing import Optional, List, Dict + +class ContactError(Exception): + pass + +class Contact: + def __init__(self): + self.contact_store = CNContactStore.alloc().init() + self.cn_contact = CNMutableContact.alloc().init() + + @classmethod + def create(cls, given_name: str, family_name: str, + email_addresses: Optional[List[tuple[str, str]]] = None, + phone_numbers: Optional[List[tuple[str, str]]] = None, + postal_address: Optional[Dict[str, str]] = None, + birthday: Optional[Dict[str, int]] = None, + image_data: Optional[bytes] = None): + contact = cls() + contact.given_name = given_name + contact.family_name = family_name + if email_addresses: + contact.email_addresses = email_addresses + if phone_numbers: + contact.phone_numbers = phone_numbers + if postal_address: + contact.set_postal_address(**postal_address) + if birthday: + contact.set_birthday(**birthday) + if image_data: + contact.image_data = image_data + return contact + + @classmethod + def from_cn_contact(cls, cn_contact: CNContact): + contact = cls() + contact.cn_contact = cn_contact.mutableCopy() + return contact + + @property + def given_name(self) -> str: + return self.cn_contact.givenName() + + @given_name.setter + def given_name(self, value: str): + self.cn_contact.setGivenName_(value) + + @property + def family_name(self) -> str: + return self.cn_contact.familyName() + + @family_name.setter + def family_name(self, value: str): + self.cn_contact.setFamilyName_(value) + + @property + def email_addresses(self) -> List[tuple[str, str]]: + return [(str(email.label()), str(email.value())) for email in self.cn_contact.emailAddresses()] + + @email_addresses.setter + def email_addresses(self, emails: List[tuple[str, str]]): + self.cn_contact.setEmailAddresses_([CNLabeledValue.labeledValueWithLabel_value_(label, email) for label, email in emails]) + + @property + def phone_numbers(self) -> List[tuple[str, str]]: + return [(str(phone.label()), str(phone.value().stringValue())) for phone in self.cn_contact.phoneNumbers()] + + @phone_numbers.setter + def phone_numbers(self, phones: List[tuple[str, str]]): + self.cn_contact.setPhoneNumbers_([CNLabeledValue.labeledValueWithLabel_value_(label, CNPhoneNumber.phoneNumberWithStringValue_(phone)) for label, phone in phones]) + + def set_postal_address(self, street: str, city: str, state: str, postal_code: str, country: Optional[str] = None): + address = CNMutablePostalAddress.alloc().init() + address.setStreet_(street) + address.setCity_(city) + address.setState_(state) + address.setPostalCode_(postal_code) + if country: + address.setCountry_(country) + self.cn_contact.setPostalAddresses_([CNLabeledValue.labeledValueWithLabel_value_(CNLabelHome, address)]) + + def set_birthday(self, day: int, month: int, year: Optional[int] = None): + birthday = NSDateComponents.alloc().init() + birthday.setDay_(day) + birthday.setMonth_(month) + if year: + birthday.setYear_(year) + self.cn_contact.setBirthday_(birthday) + + @property + def image_data(self) -> Optional[bytes]: + return self.cn_contact.imageData() + + @image_data.setter + def image_data(self, value: Optional[bytes]): + self.cn_contact.setImageData_(value) + + def save(self) -> None: + save_request = CNSaveRequest.alloc().init() + save_request.addContact_toContainerWithIdentifier_(self.cn_contact, None) + success, error = self.contact_store.executeSaveRequest_error_(save_request, None) + if not success: + raise ContactError(f"Failed to save contact: {error}") + + def remove(self) -> None: + save_request = CNSaveRequest.alloc().init() + save_request.deleteContact_(self.cn_contact) + success, error = self.contact_store.executeSaveRequest_error_(save_request, None) + if not success: + raise ContactError(f"Failed to remove contact: {error}") + + @classmethod + def fetch_contacts(cls, predicate, keys_to_fetch: List[str]): + try: + cn_contacts = cls().contact_store.unifiedContactsMatchingPredicate_keysToFetch_error_(predicate, keys_to_fetch, None) + return [cls.from_cn_contact(cn_contact) for cn_contact in cn_contacts[0]] + except objc.error as e: + raise ContactError(f"Failed to fetch contacts: {str(e)}") + + def __str__(self) -> str: + return CNContactFormatter.stringFromContact_style_(self.cn_contact, CNContactFormatterStyleFullName) + +# Example usage +if __name__ == "__main__": + try: + # contact = Contact.create( + # given_name="John", + # family_name="Doe", + # email_addresses=[(CNLabelHome, "john.doe@example.com"), (CNLabelWork, "j.doe@work.com")], + # phone_numbers=[(CNLabelHome, "(555) 123-4567")], + # postal_address={"street": "123 Apple St", "city": "Cupertino", "state": "CA", "postal_code": "95014"}, + # birthday={"day": 1, "month": 4, "year": 1988} + # ) + # contact.save() + # print(f"Saved contact: {contact}") + + # Fetch and print all contacts + all_contacts = Contact.fetch_contacts(None, [CNContactFormatter.descriptorForRequiredKeysForStyle_(CNContactFormatterStyleFullName)]) + print("\nAll contacts:") + for c in all_contacts: + print(c) + + except ContactError as e: + print(f"Error: {e}") \ No newline at end of file diff --git a/src/dspygen/pyautomator/contacts/contacts_main.py b/src/dspygen/pyautomator/contacts/contacts_main.py new file mode 100644 index 0000000..ab972b7 --- /dev/null +++ b/src/dspygen/pyautomator/contacts/contacts_main.py @@ -0,0 +1,143 @@ +# import objc +# from Contacts import CNMutableContact, CNLabeledValue, CNContactStore, CNSaveRequest, CNContact, CNPostalAddress, CNContactFormatter, CNPostalAddressFormatter, CNLabelHome, CNLabelWork +# from Foundation import NSString, NSDateComponents +# from typing import Optional, List +# +# class ContactError(Exception): +# """Custom exception for contact-related errors.""" +# pass +# +# class Contact: +# def __init__(self): +# self.cn_contact = CNMutableContact.alloc().init() +# self.store = CNContactStore.alloc().init() +# +# @classmethod +# def create(cls, given_name: str, family_name: str, email_addresses: Optional[List[tuple[str, str]]] = None, phone_numbers: Optional[List[tuple[str, str]]] = None, postal_address: Optional[dict] = None, birthday: Optional[dict] = None, image_data: Optional[bytes] = None): +# contact = cls() +# contact.given_name = given_name +# contact.family_name = family_name +# +# if email_addresses: +# contact.email_addresses = email_addresses +# +# if phone_numbers: +# contact.phone_numbers = phone_numbers +# +# if postal_address: +# contact.set_postal_address(**postal_address) +# +# if birthday: +# contact.set_birthday(**birthday) +# +# if image_data: +# contact.image_data = image_data +# +# return contact +# +# @property +# def given_name(self) -> str: +# return self.cn_contact.givenName() +# +# @given_name.setter +# def given_name(self, value: str): +# self.cn_contact.setGivenName_(value) +# +# @property +# def family_name(self) -> str: +# return self.cn_contact.familyName() +# +# @family_name.setter +# def family_name(self, value: str): +# self.cn_contact.setFamilyName_(value) +# +# @property +# def email_addresses(self) -> List[tuple[str, str]]: +# return [(str(email.label()), str(email.value())) for email in self.cn_contact.emailAddresses()] +# +# @email_addresses.setter +# def email_addresses(self, emails: List[tuple[str, str]]): +# self.cn_contact.setEmailAddresses_([CNLabeledValue.label_withValue_(label, NSString.stringWithString_(email)) for label, email in emails]) +# +# @property +# def phone_numbers(self) -> List[tuple[str, str]]: +# return [(str(phone.label()), str(phone.value().stringValue())) for phone in self.cn_contact.phoneNumbers()] +# +# @phone_numbers.setter +# def phone_numbers(self, phones: List[tuple[str, str]]): +# self.cn_contact.setPhoneNumbers_([CNLabeledValue.label_withValue_(label, CNPhoneNumber.phoneNumberWithStringValue_(phone)) for label, phone in phones]) +# +# def set_postal_address(self, street: str, city: str, state: str, postal_code: str, country: Optional[str] = None): +# address = CNMutablePostalAddress.alloc().init() +# address.setStreet_(street) +# address.setCity_(city) +# address.setState_(state) +# address.setPostalCode_(postal_code) +# if country: +# address.setCountry_(country) +# self.cn_contact.setPostalAddresses_([CNLabeledValue.label_withValue_(CNLabelHome, address)]) +# +# def set_birthday(self, day: int, month: int, year: Optional[int] = None): +# birthday = NSDateComponents.alloc().init() +# birthday.setDay_(day) +# birthday.setMonth_(month) +# if year: +# birthday.setYear_(year) +# self.cn_contact.setBirthday_(birthday) +# +# @property +# def image_data(self) -> Optional[bytes]: +# return self.cn_contact.imageData() +# +# @image_data.setter +# def image_data(self, value: Optional[bytes]): +# self.cn_contact.setImageData_(value) +# +# def save(self): +# save_request = CNSaveRequest.alloc().init() +# save_request.addContact_toContainerWithIdentifier_(self.cn_contact, None) +# try: +# self.store.executeSaveRequest_error_(save_request, None) +# except objc.error as e: +# raise ContactError(f"Failed to save contact: {str(e)}") +# +# @staticmethod +# def format_name(contact: CNContact, style: CNContactFormatterStyle) -> str: +# return str(CNContactFormatter.stringFromContact_style_(contact, style)) +# +# @staticmethod +# def format_address(address: CNPostalAddress) -> str: +# return CNPostalAddressFormatter.stringFromPostalAddress_(address) +# +# @classmethod +# def fetch_contacts(cls, predicate, keys_to_fetch: List[str]) -> List[CNContact]: +# store = CNContactStore.alloc().init() +# try: +# contacts = store.unifiedContactsMatchingPredicate_keysToFetch_error_(predicate, keys_to_fetch, None) +# return contacts +# except objc.error as e: +# raise ContactError(f"Failed to fetch contacts: {str(e)}") +# +# @classmethod +# def from_cn_contact(cls, cn_contact: CNContact) -> Contact: +# contact = cls() +# contact.cn_contact = cn_contact +# return contact +# +# def __str__(self) -> str: +# return self.format_name(self.cn_contact) +# +# # Example usage +# try: +# contact = Contact.create( +# given_name="John", +# family_name="Doe", +# email_addresses=[(CNLabelHome, "john.doe@example.com"), (CNLabelWork, "j.doe@work.com")], +# phone_numbers=[(CNLabelPhoneNumberiPhone, "(555) 123-4567")], +# postal_address={"street": "123 Apple St", "city": "Cupertino", "state": "CA", "postal_code": "95014"}, +# birthday={"day": 1, "month": 4, "year": 1988} +# ) +# contact.save() +# print(f"Saved contact: {contact}") +# except ContactError as e: +# print(e) diff --git a/src/dspygen/pyautomator/contacts/organization_contact.py b/src/dspygen/pyautomator/contacts/organization_contact.py new file mode 100644 index 0000000..b091236 --- /dev/null +++ b/src/dspygen/pyautomator/contacts/organization_contact.py @@ -0,0 +1,42 @@ +from .contact import Contact, ContactError +from Contacts import CNContactType, CNLabelWork + +class OrganizationContact(Contact): + @classmethod + def create(cls, organization_name: str, **kwargs): + contact = cls() + contact.cn_contact.setContactType_(CNContactType.CNContactTypeOrganization) + contact.organization_name = organization_name + + if 'email_addresses' in kwargs: + contact.email_addresses = kwargs['email_addresses'] + if 'phone_numbers' in kwargs: + contact.phone_numbers = kwargs['phone_numbers'] + if 'postal_address' in kwargs: + contact.set_postal_address(**kwargs['postal_address']) + if 'image_data' in kwargs: + contact.image_data = kwargs['image_data'] + + return contact + + @property + def organization_name(self) -> str: + return self.cn_contact.organizationName() + + @organization_name.setter + def organization_name(self, value: str): + self.cn_contact.setOrganizationName_(value) + +# Example usage +if __name__ == "__main__": + try: + org = OrganizationContact.create( + organization_name="Acme Corporation", + email_addresses=[(CNLabelWork, "info@acme.com")], + phone_numbers=[(CNLabelWork, "(555) 987-6543")], + postal_address={"street": "456 Tech Blvd", "city": "San Francisco", "state": "CA", "postal_code": "94105"} + ) + org.save() + print(f"Saved organization contact: {org.organization_name}") + except ContactError as e: + print(f"Error: {e}") \ No newline at end of file diff --git a/src/dspygen/pyautomator/contacts/person_contact.py b/src/dspygen/pyautomator/contacts/person_contact.py new file mode 100644 index 0000000..9a39d42 --- /dev/null +++ b/src/dspygen/pyautomator/contacts/person_contact.py @@ -0,0 +1,48 @@ +from .contact import Contact, ContactError +from Contacts import CNContactType, CNLabelHome, CNLabelWork + +class PersonContact(Contact): + @classmethod + def create(cls, given_name: str, family_name: str, **kwargs): + contact = cls() + contact.cn_contact.setContactType_(CNContactType.CNContactTypePerson) + contact.given_name = given_name + contact.family_name = family_name + + if 'email_addresses' in kwargs: + contact.email_addresses = kwargs['email_addresses'] + if 'phone_numbers' in kwargs: + contact.phone_numbers = kwargs['phone_numbers'] + if 'postal_address' in kwargs: + contact.set_postal_address(**kwargs['postal_address']) + if 'birthday' in kwargs: + contact.set_birthday(**kwargs['birthday']) + if 'image_data' in kwargs: + contact.image_data = kwargs['image_data'] + if 'job_title' in kwargs: + contact.job_title = kwargs['job_title'] + if 'department_name' in kwargs: + contact.department_name = kwargs['department_name'] + + return contact + + @property + def full_name(self) -> str: + return f"{self.given_name} {self.family_name}" + +# Example usage +if __name__ == "__main__": + try: + person = PersonContact.create( + given_name="John", + family_name="Doe", + email_addresses=[(CNLabelHome, "john.doe@example.com"), (CNLabelWork, "j.doe@work.com")], + phone_numbers=[(CNLabelHome, "(555) 123-4567")], + postal_address={"street": "123 Apple St", "city": "Cupertino", "state": "CA", "postal_code": "95014"}, + birthday={"day": 1, "month": 4, "year": 1988}, + job_title="Software Engineer" + ) + person.save() + print(f"Saved person contact: {person.full_name}") + except ContactError as e: + print(f"Error: {e}") \ No newline at end of file diff --git a/src/dspygen/pyautomator/event_kit/__init__.py b/src/dspygen/pyautomator/event_kit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dspygen/experiments/cal_apps/alarm.py b/src/dspygen/pyautomator/event_kit/alarm.py similarity index 100% rename from src/dspygen/experiments/cal_apps/alarm.py rename to src/dspygen/pyautomator/event_kit/alarm.py diff --git a/src/dspygen/experiments/cal_apps/alarm_integration_testing.py b/src/dspygen/pyautomator/event_kit/alarm_integration_testing.py similarity index 95% rename from src/dspygen/experiments/cal_apps/alarm_integration_testing.py rename to src/dspygen/pyautomator/event_kit/alarm_integration_testing.py index 42f6f0a..c924aa7 100644 --- a/src/dspygen/experiments/cal_apps/alarm_integration_testing.py +++ b/src/dspygen/pyautomator/event_kit/alarm_integration_testing.py @@ -1,8 +1,9 @@ from datetime import datetime, timedelta import inject import EventKit -from dspygen.experiments.cal_apps.reminder import Reminder -from dspygen.experiments.cal_apps.alarm import Alarm + +from dspygen.pyautomator.event_kit.alarm import Alarm +from dspygen.pyautomator.event_kit.reminder import Reminder @inject.autoparams() diff --git a/src/dspygen/experiments/cal_apps/calendar_event.py b/src/dspygen/pyautomator/event_kit/calendar_event.py similarity index 100% rename from src/dspygen/experiments/cal_apps/calendar_event.py rename to src/dspygen/pyautomator/event_kit/calendar_event.py diff --git a/src/dspygen/experiments/cal_apps/calendar_event_list.py b/src/dspygen/pyautomator/event_kit/calendar_event_list.py similarity index 97% rename from src/dspygen/experiments/cal_apps/calendar_event_list.py rename to src/dspygen/pyautomator/event_kit/calendar_event_list.py index fb42f1f..2d584b1 100644 --- a/src/dspygen/experiments/cal_apps/calendar_event_list.py +++ b/src/dspygen/pyautomator/event_kit/calendar_event_list.py @@ -3,7 +3,8 @@ from typing import List, Optional import inject -from dspygen.experiments.cal_apps.calendar_event import CalendarEvent +from dspygen.pyautomator.event_kit.calendar_event import CalendarEvent + class CalendarEventList: @inject.autoparams() diff --git a/src/dspygen/experiments/cal_apps/calendar_item.py b/src/dspygen/pyautomator/event_kit/calendar_item.py similarity index 100% rename from src/dspygen/experiments/cal_apps/calendar_item.py rename to src/dspygen/pyautomator/event_kit/calendar_item.py diff --git a/src/dspygen/experiments/cal_apps/event_store.py b/src/dspygen/pyautomator/event_kit/event_store.py similarity index 98% rename from src/dspygen/experiments/cal_apps/event_store.py rename to src/dspygen/pyautomator/event_kit/event_store.py index ec88bd5..64fa101 100644 --- a/src/dspygen/experiments/cal_apps/event_store.py +++ b/src/dspygen/pyautomator/event_kit/event_store.py @@ -114,7 +114,7 @@ def export_items_to_csv_reminder(self, filename, days=7): reminder.calendarItemIdentifier(), # Reminder ID reminder.calendar().title(), reminder.title(), - reminder.dueDate().date() if reminder.dueDate() else today, # Use today's date if no due date + reminder.dueDate() if reminder.dueDate() else today, # Use today's date if no due date reminder.priority(), 1 if reminder.isCompleted() else 0, # Use 1 for completed, 0 for not completed reminder.notes() or "", diff --git a/src/dspygen/experiments/cal_apps/recurrence_rule.py b/src/dspygen/pyautomator/event_kit/recurrence_rule.py similarity index 100% rename from src/dspygen/experiments/cal_apps/recurrence_rule.py rename to src/dspygen/pyautomator/event_kit/recurrence_rule.py diff --git a/src/dspygen/experiments/cal_apps/reminder.py b/src/dspygen/pyautomator/event_kit/reminder.py similarity index 98% rename from src/dspygen/experiments/cal_apps/reminder.py rename to src/dspygen/pyautomator/event_kit/reminder.py index 0b1286f..2d2c271 100644 --- a/src/dspygen/experiments/cal_apps/reminder.py +++ b/src/dspygen/pyautomator/event_kit/reminder.py @@ -4,14 +4,15 @@ import EventKit from Foundation import NSDateComponents from datetime import datetime, timedelta -from typing import Optional +from typing import Optional, List import inject -from dspygen.experiments.cal_apps.calendar_item import CalendarItemError, CalendarItem from dspygen.modules.generate_icalendar_module import generate_i_calendar_call from icalendar import Calendar import logging +from dspygen.pyautomator.event_kit.calendar_item import CalendarItemError, CalendarItem + logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) diff --git a/src/dspygen/experiments/cal_apps/reminder_list.py b/src/dspygen/pyautomator/event_kit/reminder_list.py similarity index 98% rename from src/dspygen/experiments/cal_apps/reminder_list.py rename to src/dspygen/pyautomator/event_kit/reminder_list.py index 24e7d6a..ed6b8fc 100644 --- a/src/dspygen/experiments/cal_apps/reminder_list.py +++ b/src/dspygen/pyautomator/event_kit/reminder_list.py @@ -4,7 +4,8 @@ from typing import Optional, List import inject from faker import Faker -from dspygen.experiments.cal_apps.reminder import Reminder, ReminderError + +from dspygen.pyautomator.event_kit.reminder import Reminder class ReminderList: diff --git a/src/dspygen/pyautomator/linkedin/__init__.py b/src/dspygen/pyautomator/linkedin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dspygen/pyautomator/linkedin/linkedin_app.py b/src/dspygen/pyautomator/linkedin/linkedin_app.py new file mode 100644 index 0000000..30d0464 --- /dev/null +++ b/src/dspygen/pyautomator/linkedin/linkedin_app.py @@ -0,0 +1,78 @@ +import json +import pyperclip +import time + +from dspygen.pyautomator.safari.safari_app import SafariApp + + +class LinkedInApp(SafariApp): + def __init__(self): + super().__init__() + + def get_profile_markdown(self, profile_url, output_file=None): + self.open_url(profile_url) + time.sleep(5) # Wait for the page to load + + script = """ + (() => { + const Safari = Application('Safari'); + const tab = Safari.windows[0].currentTab(); + + const turndownURL = "https://unpkg.com/turndown/dist/turndown.js"; + const nsURL = $.NSURL.URLWithString($(turndownURL)); + const data = $.NSData.dataWithContentsOfURL(nsURL); + const turndownSrc = $.NSString.alloc.initWithDataEncoding(data, $.NSUTF8StringEncoding).js; + + const scriptSrc = turndownSrc + + ` + var turndownService = new TurndownService(); + /* Ignore 'script' and 'nav' elements to keep + markdown cleaner */ + turndownService.remove('script'); + turndownService.remove('nav'); + + var mainContent = document.querySelector('main').innerHTML; + + // Convert to Markdown + var markdown = turndownService.turndown(mainContent); + markdown; + `; + const result = Safari.doJavaScript(`${scriptSrc}`, {in: tab}); + + // Copy the result to clipboard + const app = Application.currentApplication(); + app.includeStandardAdditions = true; + app.setTheClipboardTo(result); + })() + """ + self.execute_jxa(script) + + # Give some time for the clipboard to be updated + time.sleep(0.5) + + # Get the result from the clipboard + markdown_content = pyperclip.paste() + + if output_file: + with open(output_file, 'w', encoding='utf-8') as f: + f.write(markdown_content) + print(f"LinkedIn profile content saved to: {output_file}") + + return markdown_content + + +def main(): + linkedin = LinkedInApp() + linkedin.activate_app() + + # Example usage + profile_url = "https://www.linkedin.com/in/seanchatman/" + output_file = "linkedin_profile.md" + + markdown = linkedin.get_profile_markdown(profile_url, output_file=output_file) + print(f"Converted LinkedIn profile length: {len(markdown)}") + print(f"LinkedIn profile content saved to: {output_file}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/dspygen/pyautomator/linkedin/linkedin_profile.md b/src/dspygen/pyautomator/linkedin/linkedin_profile.md new file mode 100644 index 0000000..58d75c4 --- /dev/null +++ b/src/dspygen/pyautomator/linkedin/linkedin_profile.md @@ -0,0 +1,486 @@ +![Background Image](https://media.licdn.com/dms/image/v2/D5616AQFKLbJUVpCajQ/profile-displaybackgroundimage-shrink_350_1400/profile-displaybackgroundimage-shrink_350_1400/0/1719349341699?e=1730332800&v=beta&t=u11XKIGSXRADGJA8DxlCVlvsaFsWSOZCiECRtPAVIbo) + +![πŸ€– Sean Chatman πŸ€–, #OPEN_TO_WORK](https://media.licdn.com/dms/image/v2/D5635AQGO10TJ7Zrrtw/profile-framedphoto-shrink_200_200/profile-framedphoto-shrink_200_200/0/1719333832284?e=1725584400&v=beta&t=UbnMseiO3ZK-A58hXSpCs0wbSfgyDzGksFJ1IjKz7tw) + +[](/in/seanchatman/edit/intro/?profileFormEntryPoint=PROFILE_SECTION) + +[ + +πŸ€– Sean Chatman πŸ€– +================== + +](/in/seanchatman/overlay/about-this-profile/) + +(He/Him) + +Available for Distinguished Front End Generative AI Web Development + +Los Angeles, California, United States [Contact info](/in/seanchatman/overlay/contact-info/) + +**[linkedin.com/seanchatman](https://linkedin.com/seanchatman)** + +* [21,532 followers](/mynetwork/network-manager/people-follow/followers/) +* [500+ connections](/mynetwork/invite-connect/connections/) + +Open to + +Add profile section + +Enhance profile + +More + +* Send profile in a message + +* Save to PDF + +* About this profile About this profile + + +Enhance profile + +* [ + +### **Open to work** + +Staff Software Engineer, Senior Staff Engineer, Principal Software Engineer, Distinguished Software Engineer and Frontend Developer roles + +Show details + +](https://www.linkedin.com/in/seanchatman/opportunities/job-opportunities/details?profileUrn=urn%3Ali%3Afs_normalized_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4&trk=opento_sprofile_details) + +[Edit](https://www.linkedin.com/in/seanchatman/opportunities/job-opportunities/edit?origin=PROFILE_TOP_CARD&trk=opento_sprofile_topcard) + +* [ + +**Share that you’re hiring** and attract qualified candidates. + +Get started + +](https://www.linkedin.com/in/seanchatman/opportunities/hiring-opportunities/onboarding) + + +](https://www.linkedin.com/dashboard) + +ResourcesResources +------------------ + +Private to you Private to you + +* [ + +](https://www.linkedin.com/mynetwork/discover-hub) + +[ + +My networkMy network + +See and manage your connections and interests.See and manage your connections and interests.](https://www.linkedin.com/mynetwork/discover-hub) + +* [ + +](https://www.linkedin.com/my-items) + +[ + +Saved itemsSaved items + +Keep track of your jobs, courses, and articles.Keep track of your jobs, courses, and articles.](https://www.linkedin.com/my-items) + + +[Show all 4 resources + +](https://www.linkedin.com/in/seanchatman/details/resources?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + +AboutAbout +---------- + +[ + +](https://www.linkedin.com/in/seanchatman/add-edit/SUMMARY/?profileFormEntryPoint=PROFILE_SECTION&trackingId=ukFXqg2zTmCdkzZPr6Jgfg%3D%3D) + +With over 24 years of experience in software engineering, I have cultivated a deep expertise in front-end web development, generative AI, and full-stack solutions. My professional journey has been marked by continuous learning, innovation, and a commitment to excellence. + +Recent Experience: + +As a consultant and contractor doing Distinguished Front End Generative AI Web Development, I focus on harnessing the power of generative AI to revolutionize front-end web development. My role involves creating dynamic and responsive prototypes, showcasing the immense potential of AI-driven technologies. I design and implement AI tools that automated the development process, enhancing user experience and delivering sophisticated interfaces. My work is not just about coding; it is about understanding stakeholder needs, solving complex problems, and staying ahead of the latest technological trends. + +Previous Roles: + +At Intuit, as a Staff Software Engineer in Artificial Intelligence, Analytics, and Data, I played a key role in integrating AI and analytics into business applications, driving innovation and efficiency. My contributions ranged from architectural design to full-stack development, using technologies like React, Node.js, MongoDB, and OpenAI's GPT models. + +My journey also includes significant roles at companies like Method Studios, AT&T, Playsino, Riot Games, and Neopets, where I developed and led projects that spanned from web development and software architecture to full-stack engineering and DevOps. Each role allowed me to refine my skills and contribute to cutting-edge projects, mentoring teams, and driving technological advancements. + +Skills and Expertise: + +\- Front-End Development: Proficient in HTML5, CSS, JavaScript, React, Angular, and Vue. + +\- Generative AI: Expertise in using OpenAI’s GPT models to enhance web development and create intelligent applications. + +\- Full-Stack Development: Experienced with Node.js, MongoDB, MySQL, Docker, and Kubernetes. + +\- Prototyping and UX: Skilled in creating dynamic, responsive prototypes and ensuring optimal user experience. + +\- Collaboration and Problem-Solving: Strong track record of working with cross-functional teams to deliver tailored solutions that meet strategic objectives. + +My professional ethos is rooted in the belief that technology should empower and transform. I am passionate about exploring new frontiers in AI and web development, and I look forward to leveraging my skills and experience in a full-time role where I can continue to make a meaningful impact.With over 24 years of experience in software engineering, I have cultivated a deep expertise in front-end web development, generative AI, and full-stack solutions. My professional journey has been marked by continuous learning, innovation, and a commitment to excellence. Recent Experience: As a consultant and contractor doing Distinguished Front End Generative AI Web Development, I focus on harnessing the power of generative AI to revolutionize front-end web development. My role involves creating dynamic and responsive prototypes, showcasing the immense potential of AI-driven technologies. I design and implement AI tools that automated the development process, enhancing user experience and delivering sophisticated interfaces. My work is not just about coding; it is about understanding stakeholder needs, solving complex problems, and staying ahead of the latest technological trends. Previous Roles: At Intuit, as a Staff Software Engineer in Artificial Intelligence, Analytics, and Data, I played a key role in integrating AI and analytics into business applications, driving innovation and efficiency. My contributions ranged from architectural design to full-stack development, using technologies like React, Node.js, MongoDB, and OpenAI's GPT models. My journey also includes significant roles at companies like Method Studios, AT&T, Playsino, Riot Games, and Neopets, where I developed and led projects that spanned from web development and software architecture to full-stack engineering and DevOps. Each role allowed me to refine my skills and contribute to cutting-edge projects, mentoring teams, and driving technological advancements. Skills and Expertise: - Front-End Development: Proficient in HTML5, CSS, JavaScript, React, Angular, and Vue. - Generative AI: Expertise in using OpenAI’s GPT models to enhance web development and create intelligent applications. - Full-Stack Development: Experienced with Node.js, MongoDB, MySQL, Docker, and Kubernetes. - Prototyping and UX: Skilled in creating dynamic, responsive prototypes and ensuring optimal user experience. - Collaboration and Problem-Solving: Strong track record of working with cross-functional teams to deliver tailored solutions that meet strategic objectives. My professional ethos is rooted in the belief that technology should empower and transform. I am passionate about exploring new frontiers in AI and web development, and I look forward to leveraging my skills and experience in a full-time role where I can continue to make a meaningful impact. + +* Top skillsTop skills + +Artificial Intelligence (AI) β€’ TypeScript β€’ Python (Programming Language) β€’ Generative AI β€’ Revenue & Profit GrowthArtificial Intelligence (AI) β€’ TypeScript β€’ Python (Programming Language) β€’ Generative AI β€’ Revenue & Profit Growth + +[ + +](https://www.linkedin.com/in/seanchatman/overlay/top-skills-details?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + + +ServicesServices +---------------- + +[ + +](https://www.linkedin.com/in/seanchatman/opportunities/services/edit?servicePageVanityName=540a50330087461445) + +* * **Web Development β€’ Application Development β€’ Custom Software Development β€’ SaaS Development****Web Development β€’ Application Development β€’ Custom Software Development β€’ SaaS Development** + + + +[Show all + +](https://www.linkedin.com/services/page/540a50330087461445) + +FeaturedFeatured +---------------- + +[ + +](https://www.linkedin.com/in/seanchatman/details/featured?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + +ExperienceExperience +-------------------- + +[ + +](https://www.linkedin.com/in/seanchatman/details/experience?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + +* [ + +](https://www.linkedin.com/search/results/all/?keywords=Consulting) + +Distinguished Front End Generative AI Web Development ContractorDistinguished Front End Generative AI Web Development Contractor + +Consulting Β· ContractConsulting Β· Contract May 2023 - Aug 2024 Β· 1 yr 4 mosMay 2023 to Aug 2024 Β· 1 yr 4 mos Los Angeles County, California, United States Β· RemoteLos Angeles County, California, United States Β· Remote + +* * I focus on developing innovative solutions leveraging generative AI for front-end web development. My work has involved creating dynamic and responsive prototypes to showcase the potential of AI-driven front-end technologies. + +Key Achievements: + +\- AI-Powered Front-End Prototyping: Designed and implemented AI-driven tools that automated the creation of dynamic and responsive web prototypes. + +\- Advanced Front-End Development: Utilized cutting-edge web frameworks and technologies, including React, Vue, HTML5, CSS, Typescript, and JavaScript, to develop sophisticated user interfaces. + +\- Integration of Generative AI: Applied OpenAI's GPT-3.5 and GPT-4 to enhance user experience and streamline development processes through intelligent automation. + +\- Collaborative Problem Solving: Worked closely with stakeholders to understand their needs and deliver tailored prototype solutions that aligned with their strategic objectives. + +\- Continuous Learning and Adaptation: Stayed up-to-date with the latest trends and technologies in AI and front-end development to ensure the delivery of state-of-the-art solutions.I focus on developing innovative solutions leveraging generative AI for front-end web development. My work has involved creating dynamic and responsive prototypes to showcase the potential of AI-driven front-end technologies. Key Achievements: - AI-Powered Front-End Prototyping: Designed and implemented AI-driven tools that automated the creation of dynamic and responsive web prototypes. - Advanced Front-End Development: Utilized cutting-edge web frameworks and technologies, including React, Vue, HTML5, CSS, Typescript, and JavaScript, to develop sophisticated user interfaces. - Integration of Generative AI: Applied OpenAI's GPT-3.5 and GPT-4 to enhance user experience and streamline development processes through intelligent automation. - Collaborative Problem Solving: Worked closely with stakeholders to understand their needs and deliver tailored prototype solutions that aligned with their strategic objectives. - Continuous Learning and Adaptation: Stayed up-to-date with the latest trends and technologies in AI and front-end development to ensure the delivery of state-of-the-art solutions. + + + +* [ + +![Intuit logo](https://media.licdn.com/dms/image/v2/C560BAQFTpF8uneqScw/company-logo_100_100/company-logo_100_100/0/1661446146222/intuit_logo?e=1733356800&v=beta&t=EQpw1PuBVwJfUOeVzRedGKdCOzfsc4tp0bo3BPYyIfE) + + + +](https://www.linkedin.com/company/1666/) + +[ + +IntuitIntuit + +2 yrs 5 mos2 yrs 5 mos](https://www.linkedin.com/company/1666/) + +* [ + +Staff Software Engineer - Artificial Intelligence, Analytics, and DataStaff Software Engineer - Artificial Intelligence, Analytics, and Data + +Full-timeFull-time Dec 2021 - Jun 2023 Β· 1 yr 7 mosDec 2021 to Jun 2023 Β· 1 yr 7 mos Los Angeles, California, United States Β· RemoteLos Angeles, California, United States Β· Remote](https://www.linkedin.com/company/1666/) + +* * In my role as a Staff Software Engineer at Intuit, I focused on harnessing the power of AI, analytics, and data to innovate and streamline business processes. My contributions were integral to the architectural design and development of advanced business applications, utilizing my expertise in modern web frameworks and full-stack technologies. + +Key responsibilities and achievements include: + +Innovation through AI, Analytics, and Data: +\- Spearheaded projects to integrate AI and analytics into business applications, driving efficiency and innovation. + +Architectural Design and Development: +\- Played a critical role in the planning and execution of application architectures, ensuring scalability and performance. + +Expertise in Cutting-Edge Technologies: +\- Demonstrated proficiency in a range of technologies including React for front-end development, and Node.js, MongoDB, MySQL for backend infrastructure. OpenAI API GPT-3.5 and GPT-4 for generative AI features. + +Mentorship and Collaboration: +\- Actively engaged in mentoring team members, sharing expertise in both technical and soft skills to promote team growth and cohesion. + +Software Solution Excellence: +\- Contributed significantly to the development of robust, scalable, and innovative software solutions that met and exceeded project goals. + +My tenure at Intuit not only allowed me to contribute to the technological advancement of the company but also enabled me to foster a culture of learning and collaboration. I am proud to have played a part in driving Intuit's mission forward through my dedication to technological innovation and team development.In my role as a Staff Software Engineer at Intuit, I focused on harnessing the power of AI, analytics, and data to innovate and streamline business processes. My contributions were integral to the architectural design and development of advanced business applications, utilizing my expertise in modern web frameworks and full-stack technologies. Key responsibilities and achievements include: Innovation through AI, Analytics, and Data: - Spearheaded projects to integrate AI and analytics into business applications, driving efficiency and innovation. Architectural Design and Development: - Played a critical role in the planning and execution of application architectures, ensuring scalability and performance. Expertise in Cutting-Edge Technologies: - Demonstrated proficiency in a range of technologies including React for front-end development, and Node.js, MongoDB, MySQL for backend infrastructure. OpenAI API GPT-3.5 and GPT-4 for generative AI features. Mentorship and Collaboration: - Actively engaged in mentoring team members, sharing expertise in both technical and soft skills to promote team growth and cohesion. Software Solution Excellence: - Contributed significantly to the development of robust, scalable, and innovative software solutions that met and exceeded project goals. My tenure at Intuit not only allowed me to contribute to the technological advancement of the company but also enabled me to foster a culture of learning and collaboration. I am proud to have played a part in driving Intuit's mission forward through my dedication to technological innovation and team development. + + +* * [ + +**Python (Programming Language), Artificial Intelligence (AI) and +3 skills** + + + + + +](https://www.linkedin.com/in/seanchatman/overlay/urn:li:fsd_profilePosition:\(ACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4,1881903597\)/skill-associations-details?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + + + +* [ + +Full-Stack Software Consultant - Artificial Intelligence, Analytics, and DataFull-Stack Software Consultant - Artificial Intelligence, Analytics, and Data + +ContractContract Feb 2021 - Dec 2021 Β· 11 mosFeb 2021 to Dec 2021 Β· 11 mos Los Angeles Metropolitan AreaLos Angeles Metropolitan Area](https://www.linkedin.com/company/1666/) + +* * In my role as a Full-Stack Software Consultant at Intuit, I concentrated on utilizing my extensive expertise in HTML5, CSS, GraphQL, TypeScript, and JavaScript to develop full-stack solutions. My responsibilities involved working closely with various teams to deliver data-centric and AI-powered applications, optimizing them for performance and user experience. I engaged in hands-on development, ensuring that the client-side interfaces were seamlessly integrated with server-side functionalities. My approach combined technical skill with a deep understanding of business requirements, aligning software solutions with strategic objectives. + +Highlights: +\- Developed AI and analytics-driven full-stack solutions. +\- Expertise in HTML5, CSS, GraphQL, TypeScript, and JavaScript. +\- Ensured optimal performance and user experience in application development. +\- Integrated client-side and server-side functionalities effectively. +\- Delivered solutions aligned with business strategies and user needs.In my role as a Full-Stack Software Consultant at Intuit, I concentrated on utilizing my extensive expertise in HTML5, CSS, GraphQL, TypeScript, and JavaScript to develop full-stack solutions. My responsibilities involved working closely with various teams to deliver data-centric and AI-powered applications, optimizing them for performance and user experience. I engaged in hands-on development, ensuring that the client-side interfaces were seamlessly integrated with server-side functionalities. My approach combined technical skill with a deep understanding of business requirements, aligning software solutions with strategic objectives. Highlights: - Developed AI and analytics-driven full-stack solutions. - Expertise in HTML5, CSS, GraphQL, TypeScript, and JavaScript. - Ensured optimal performance and user experience in application development. - Integrated client-side and server-side functionalities effectively. - Delivered solutions aligned with business strategies and user needs. + + +* * [ + +**HTML5, GraphQL and +3 skills** + + + + + +](https://www.linkedin.com/in/seanchatman/overlay/urn:li:fsd_profilePosition:\(ACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4,1776957884\)/skill-associations-details?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + + + + +* [ + +![Method Studios logo](https://media.licdn.com/dms/image/v2/C4E0BAQH7_tAzhIXOzg/company-logo_100_100/company-logo_100_100/0/1674758073406?e=1733356800&v=beta&t=mpDlxkCmvv0PeMpxYeyN3up5Vc7of1qjvrWvUzkzkEs) + + + +](https://www.linkedin.com/company/22437/) + +Principal Engineer (Web)Principal Engineer (Web) + +Method Studios Β· Full-timeMethod Studios Β· Full-time Feb 2018 - Jun 2020 Β· 2 yrs 5 mosFeb 2018 to Jun 2020 Β· 2 yrs 5 mos Greater Los Angeles AreaGreater Los Angeles Area + +* * At Method Studios, as a Principal Engineer focusing on web technologies, I provided extensive expertise in the architectural design and development of sophisticated business applications. I was deeply involved in hands-on development and architectural decisions, utilizing major web frameworks like React, Angular, and Vue to deliver robust and scalable web solutions. My role extended to full-stack infrastructure development, where I demonstrated proficiency with technologies such as Node.js, MongoDB, MySQL, Docker, and Kubernetes. Beyond technical responsibilities, I played a pivotal role in mentoring junior colleagues, sharing knowledge and guiding their professional growth in a dynamic and fast-paced environment. My contributions at Method Studios were characterized by a blend of technical acumen, leadership, and a commitment to fostering a collaborative and learning-driven work culture. + +Highlights: +\- Led the architectural design and development of advanced business applications using React, Angular, and Vue. +\- Managed full-stack infrastructure projects involving Node.js, MongoDB, MySQL, Docker, and Kubernetes. +\- Instrumental in introducing and implementing modern web technologies and frameworks. +\- Played a key role in mentoring and guiding junior engineers, enhancing team capabilities and knowledge. +\- Contributed to building scalable and efficient web solutions, aligning with business goals and user needs.At Method Studios, as a Principal Engineer focusing on web technologies, I provided extensive expertise in the architectural design and development of sophisticated business applications. I was deeply involved in hands-on development and architectural decisions, utilizing major web frameworks like React, Angular, and Vue to deliver robust and scalable web solutions. My role extended to full-stack infrastructure development, where I demonstrated proficiency with technologies such as Node.js, MongoDB, MySQL, Docker, and Kubernetes. Beyond technical responsibilities, I played a pivotal role in mentoring junior colleagues, sharing knowledge and guiding their professional growth in a dynamic and fast-paced environment. My contributions at Method Studios were characterized by a blend of technical acumen, leadership, and a commitment to fostering a collaborative and learning-driven work culture. Highlights: - Led the architectural design and development of advanced business applications using React, Angular, and Vue. - Managed full-stack infrastructure projects involving Node.js, MongoDB, MySQL, Docker, and Kubernetes. - Instrumental in introducing and implementing modern web technologies and frameworks. - Played a key role in mentoring and guiding junior engineers, enhancing team capabilities and knowledge. - Contributed to building scalable and efficient web solutions, aligning with business goals and user needs. + + +* * [ + +**Docker, React.js and +3 skills** + + + + + +](https://www.linkedin.com/in/seanchatman/overlay/urn:li:fsd_profilePosition:\(ACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4,1321753592\)/skill-associations-details?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + + + +* [ + +![AT&T logo](https://media.licdn.com/dms/image/v2/C560BAQE6Wr9RUG3OuA/company-logo_100_100/company-logo_100_100/0/1631399436556/att_logo?e=1733356800&v=beta&t=AY0BRAvxQyK1HkHN_YcdxRkW4ITboKTUg6pf1-zgMys) + + + +](https://www.linkedin.com/company/1052/) + +Lead Software EngineerLead Software Engineer + +AT&TAT&T Feb 2016 - Nov 2017 Β· 1 yr 10 mosFeb 2016 to Nov 2017 Β· 1 yr 10 mos Greater Los Angeles AreaGreater Los Angeles Area + +* * In my role as Lead Software Engineer at AT&T, I specialized in AngularJS 2, focusing primarily on developing robust client-side applications. My responsibilities included the creation and integration of modules and components to build highly functional and performance-oriented user interfaces. I worked closely with artistic designers, translating their HTML templates into dynamic and responsive web designs, complete with animations and CSS stylings. Collaborating effectively with back-end development teams, I ensured seamless API integrations and data communication. My approach to development was marked by a commitment to creating intuitive, efficient, and visually appealing user experiences, backed by solid technical execution and team coordination. + +Highlights: +\- Spearheaded the development of client-side applications using AngularJS 2, enhancing user experience and application performance. +\- Translated artistic design concepts into responsive web interfaces, incorporating animations and CSS stylings. +\- Led module and component creation, integrating them to form cohesive and functional applications. +\- Facilitated effective collaboration between front-end and back-end teams for seamless API integrations. +\- Focused on delivering high-quality user interfaces with an emphasis on performance and user engagement.In my role as Lead Software Engineer at AT&T, I specialized in AngularJS 2, focusing primarily on developing robust client-side applications. My responsibilities included the creation and integration of modules and components to build highly functional and performance-oriented user interfaces. I worked closely with artistic designers, translating their HTML templates into dynamic and responsive web designs, complete with animations and CSS stylings. Collaborating effectively with back-end development teams, I ensured seamless API integrations and data communication. My approach to development was marked by a commitment to creating intuitive, efficient, and visually appealing user experiences, backed by solid technical execution and team coordination. Highlights: - Spearheaded the development of client-side applications using AngularJS 2, enhancing user experience and application performance. - Translated artistic design concepts into responsive web interfaces, incorporating animations and CSS stylings. - Led module and component creation, integrating them to form cohesive and functional applications. - Facilitated effective collaboration between front-end and back-end teams for seamless API integrations. - Focused on delivering high-quality user interfaces with an emphasis on performance and user engagement. + + +* * [ + +**Angular2, Engineering Consulting and +1 skill** + + + + + +](https://www.linkedin.com/in/seanchatman/overlay/urn:li:fsd_profilePosition:\(ACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4,877834442\)/skill-associations-details?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + + + +* [ + +![Playsino logo](https://media.licdn.com/dms/image/v2/C4E0BAQE1X1QymsQTeQ/company-logo_100_100/company-logo_100_100/0/1631356177245?e=1733356800&v=beta&t=RFt8P2shkDsymklWrz5WNC6texs-CK-7Ivxy5stfHrA) + + + +](https://www.linkedin.com/company/2548549/) + +Software ArchitectSoftware Architect + +PlaysinoPlaysino Jun 2012 - May 2015 Β· 3 yrsJun 2012 to May 2015 Β· 3 yrs Santa Monica,CASanta Monica,CA + +* * As a Software Architect at Playsino, I was instrumental in the development of 'Bingo Around the World.' My role involved utilizing a diverse technology stack including Jenkins, Java, Intellij, JavaScript, JSON, REST, EC2, Python, XHTML, Facebook's Open Graph, and Linux. I led the Client Engineering team, fostering a collaborative environment and introducing new technologies. My approach to software architecture was marked by a keen focus on the 'why' behind every project, ensuring that each solution was not only technically sound but also purpose-driven and user-centric. My mentorship of junior team members, including interns like Clinton Jake VanSciver, was geared towards nurturing their potential and instilling a similar approach to technology and project execution. This period was marked by significant contributions to the foundational aspects of eSports and a constant engagement with emerging technologies to keep ahead of industry trends. + +Highlights: +\- Led the architectural development of 'Bingo Around the World,' a key project at Playsino, leveraging advanced Java and web technologies. +\- Managed the Client Engineering team, introducing and integrating new technologies to enhance project outcomes. +\- Played a mentorship role, providing guidance and technical leadership to junior team members and interns. +\- Maintained a focus on purpose-driven development, ensuring that projects delivered both technical excellence and meaningful user experiences. +\- Actively engaged with new and emerging technologies, contributing to the foundational growth of eSports and maintaining a forward-thinking approach in software architecture.As a Software Architect at Playsino, I was instrumental in the development of 'Bingo Around the World.' My role involved utilizing a diverse technology stack including Jenkins, Java, Intellij, JavaScript, JSON, REST, EC2, Python, XHTML, Facebook's Open Graph, and Linux. I led the Client Engineering team, fostering a collaborative environment and introducing new technologies. My approach to software architecture was marked by a keen focus on the 'why' behind every project, ensuring that each solution was not only technically sound but also purpose-driven and user-centric. My mentorship of junior team members, including interns like Clinton Jake VanSciver, was geared towards nurturing their potential and instilling a similar approach to technology and project execution. This period was marked by significant contributions to the foundational aspects of eSports and a constant engagement with emerging technologies to keep ahead of industry trends. Highlights: - Led the architectural development of 'Bingo Around the World,' a key project at Playsino, leveraging advanced Java and web technologies. - Managed the Client Engineering team, introducing and integrating new technologies to enhance project outcomes. - Played a mentorship role, providing guidance and technical leadership to junior team members and interns. - Maintained a focus on purpose-driven development, ensuring that projects delivered both technical excellence and meaningful user experiences. - Actively engaged with new and emerging technologies, contributing to the foundational growth of eSports and maintaining a forward-thinking approach in software architecture. + + +* * [ + +**Software Architecture, PHP and +1 skill** + + + + + +](https://www.linkedin.com/in/seanchatman/overlay/urn:li:fsd_profilePosition:\(ACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4,291072436\)/skill-associations-details?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + + + + +[Show all 19 experiences + +](https://www.linkedin.com/in/seanchatman/details/experience?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + +EducationEducation +------------------ + +[ + +](https://www.linkedin.com/in/seanchatman/add-edit/EDUCATION/?profileFormEntryPoint=PROFILE_SECTION&trackingId=et2oXYJcRYyP5jVbhA1aWw%3D%3D&desktopBackground=MAIN_PROFILE) + +[ + +](https://www.linkedin.com/in/seanchatman/details/education?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + +* [ + +![Santa Monica College logo](https://media.licdn.com/dms/image/v2/C560BAQEiJNL1oWf50g/company-logo_100_100/company-logo_100_100/0/1631394144823?e=1733356800&v=beta&t=TRwsaB5AkXNTvyZavVI-1mQ21rUCe62WP6TEW2qlKOE) + + + +](https://www.linkedin.com/company/165849/) + +[ + +Santa Monica CollegeSanta Monica College + + + + + + + +](https://www.linkedin.com/company/165849/) + + +VolunteeringVolunteering +------------------------ + +[ + +](https://www.linkedin.com/in/seanchatman/add-edit/VOLUNTEER/?profileFormEntryPoint=PROFILE_SECTION&trackingId=2WXZyRFLQvaWBnBDkYSqfQ%3D%3D&desktopBackground=MAIN_PROFILE) + +[ + +](https://www.linkedin.com/in/seanchatman/details/volunteering-experiences?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + +* [ + +![Santa Monica Chamber of Commerce logo](https://media.licdn.com/dms/image/v2/C560BAQHy5uL-2hOiVQ/company-logo_100_100/company-logo_100_100/0/1631301850115?e=1733356800&v=beta&t=wJOAf5Wh-dFeNdHm8GTT7ALgRKGkM5c-rIemjcdC39Y) + + + +](https://www.linkedin.com/company/121150/) + +Executive Volunteer Executive Volunteer + +Santa Monica Chamber of CommerceSanta Monica Chamber of Commerce Jun 2015 - Present Β· 9 yrs 3 mosJun 2015 - Present Β· 9 yrs 3 mos Economic EmpowermentEconomic Empowerment + + +SkillsSkills +------------ + +[ + +](https://www.linkedin.com/in/seanchatman/add-edit/SKILL_AND_ASSOCIATION/?profileFormEntryPoint=PROFILE_SECTION&trackingId=5e7a1aIVQS%2BkzA%2BghpYj%2Fw%3D%3D&desktopBackground=MAIN_PROFILE) + +[ + +](https://www.linkedin.com/in/seanchatman/details/skills?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) + +* [ + +OpenAIOpenAI + + + + + + + +](https://www.linkedin.com/search/results/all/?keywords=OpenAI&origin=PROFILE_PAGE_SKILL_NAVIGATION) + +* * ![](https://media.licdn.com/dms/image/v2/C560BAQFTpF8uneqScw/company-logo_100_100/company-logo_100_100/0/1661446146222/intuit_logo?e=1733356800&v=beta&t=EQpw1PuBVwJfUOeVzRedGKdCOzfsc4tp0bo3BPYyIfE) + + +Staff Software Engineer - Artificial Intelligence, Analytics, and Data at IntuitStaff Software Engineer - Artificial Intelligence, Analytics, and Data at Intuit + + +* [ + +Generative AIGenerative AI + + + + + + + +](https://www.linkedin.com/search/results/all/?keywords=Generative+AI&origin=PROFILE_PAGE_SKILL_NAVIGATION) + +* * ![](https://media.licdn.com/dms/image/v2/C560BAQFTpF8uneqScw/company-logo_100_100/company-logo_100_100/0/1661446146222/intuit_logo?e=1733356800&v=beta&t=EQpw1PuBVwJfUOeVzRedGKdCOzfsc4tp0bo3BPYyIfE) + + +Staff Software Engineer - Artificial Intelligence, Analytics, and Data at IntuitStaff Software Engineer - Artificial Intelligence, Analytics, and Data at Intuit + + + +[Show all 47 skills + +](https://www.linkedin.com/in/seanchatman/details/skills?profileUrn=urn%3Ali%3Afsd_profile%3AACoAAAqzEvcBM9FAJG6m49iNO_ecL06RhSjhcQ4) \ No newline at end of file diff --git a/src/dspygen/pyautomator/linkedin/sales_nav_app.py b/src/dspygen/pyautomator/linkedin/sales_nav_app.py new file mode 100644 index 0000000..5483378 --- /dev/null +++ b/src/dspygen/pyautomator/linkedin/sales_nav_app.py @@ -0,0 +1,142 @@ +import json +import pyperclip +import time +import logging +import pandas as pd + +from dspygen.pyautomator.safari.safari_app import SafariApp + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +class SalesNavApp(SafariApp): + def __init__(self): + super().__init__() + self.base_url = "https://www.linkedin.com/sales" + self.connections = None + + def login(self, username, password): + self.open_url(f"{self.base_url}/login") + time.sleep(3) # Wait for page to load + + script = f""" + (() => {{ + document.getElementById('username').value = '{username}'; + document.getElementById('password').value = '{password}'; + document.querySelector('form.login__form').submit(); + }})() + """ + self.execute_jxa(script) + time.sleep(5) # Wait for login to complete + + def search_leads(self, query): + self.open_url(f"{self.base_url}/search/people") + time.sleep(3) # Wait for page to load + + script = f""" + (() => {{ + const searchInput = document.querySelector('input[placeholder="Search by keyword"]'); + searchInput.value = '{query}'; + searchInput.dispatchEvent(new Event('input', {{ bubbles: true }})); + document.querySelector('button[aria-label="Search"]').click(); + }})() + """ + self.execute_jxa(script) + time.sleep(5) # Wait for search results to load + + def get_search_results(self, num_results=10): + script = f""" + (() => {{ + const results = Array.from(document.querySelectorAll('.search-results__result-item')).slice(0, {num_results}); + return JSON.stringify(results.map(result => {{ + const nameElement = result.querySelector('.result-lockup__name'); + const titleElement = result.querySelector('.result-lockup__highlight-keyword'); + const companyElement = result.querySelector('.result-lockup__position-company'); + return {{ + name: nameElement ? nameElement.textContent.trim() : '', + title: titleElement ? titleElement.textContent.trim() : '', + company: companyElement ? companyElement.textContent.trim() : '' + }}; + }})); + }})() + """ + result = self.execute_jxa(script) + return json.loads(result) + + def send_connection_request(self, lead_index): + script = f""" + (() => {{ + const connectButtons = document.querySelectorAll('button[aria-label="Connect"]'); + if (connectButtons[{lead_index}]) {{ + connectButtons[{lead_index}].click(); + setTimeout(() => {{ + const sendButton = document.querySelector('button[aria-label="Send now"]'); + if (sendButton) {{ + sendButton.click(); + return "Connection request sent"; + }} + return "Failed to send connection request"; + }}, 1000); + }} + return "Connect button not found"; + }})() + """ + return self.execute_jxa(script) + + def import_connections(self, file_path): + try: + self.connections = pd.read_csv(file_path) + logger.info(f"Imported {len(self.connections)} connections from {file_path}") + except Exception as e: + logger.error(f"Error importing connections: {e}") + + def compare_results_with_connections(self, search_results): + if self.connections is None: + logger.warning("Connections not imported. Please import connections first.") + return search_results + + for result in search_results: + match = self.connections[self.connections['Full Name'].str.lower() == result['name'].lower()] + if not match.empty: + result['connected'] = True + result['connected_on'] = match.iloc[0]['Connected On'] + else: + result['connected'] = False + result['connected_on'] = None + + return search_results + +def main(): + sales_nav = SalesNavApp() + sales_nav.activate_app() + + # Import connections + connections_file = "/Users/sac/Downloads/Complete_LinkedInDataExport_08-21-2024/21KLinkedInConnections.csv" + sales_nav.import_connections(connections_file) + + # Example usage + username = "your_username" + password = "your_password" + sales_nav.login(username, password) + + search_query = "Software Engineer" + sales_nav.search_leads(search_query) + + results = sales_nav.get_search_results(5) + results_with_connection_status = sales_nav.compare_results_with_connections(results) + + print("Search Results:") + for i, result in enumerate(results_with_connection_status): + connection_status = "Connected" if result['connected'] else "Not connected" + connected_on = f" (Connected on: {result['connected_on']})" if result['connected'] else "" + print(f"{i+1}. {result['name']} - {result['title']} at {result['company']} - {connection_status}{connected_on}") + + # Uncomment to send a connection request to the first non-connected result + # for result in results_with_connection_status: + # if not result['connected']: + # response = sales_nav.send_connection_request(results_with_connection_status.index(result)) + # print(response) + # break + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/dspygen/pyautomator/reminders/__init__.py b/src/dspygen/pyautomator/reminders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dspygen/experiments/cal_apps/reminder_app.py b/src/dspygen/pyautomator/reminders/reminder_app.py similarity index 94% rename from src/dspygen/experiments/cal_apps/reminder_app.py rename to src/dspygen/pyautomator/reminders/reminder_app.py index f2f1028..7e66c09 100644 --- a/src/dspygen/experiments/cal_apps/reminder_app.py +++ b/src/dspygen/pyautomator/reminders/reminder_app.py @@ -6,17 +6,20 @@ import csv import pandas as pd -from dspygen.experiments.cal_apps.event_store import EventStore -from dspygen.experiments.cal_apps.reminder_list import ReminderList +from dspygen.pyautomator.base_app import BaseApp +from dspygen.pyautomator.event_kit.event_store import EventStore +from dspygen.pyautomator.event_kit.reminder import Reminder +from dspygen.pyautomator.event_kit.reminder_list import ReminderList from dspygen.rm.data_retriever import DataRetriever from datetime import datetime, timedelta -from dspygen.experiments.cal_apps.reminder import Reminder from dspygen.modules.df_sql_module import dfsql_call from dspygen.modules.generate_icalendar_module import generate_i_calendar_call -class ReminderApp: + +class RemindersApp(BaseApp): @inject.autoparams() def __init__(self, event_store: EventKit.EKEventStore): + super().__init__("Reminders") self.event_store = event_store self.lists: List[ReminderList] = [] self._load_existing_lists() @@ -51,7 +54,7 @@ def query(self, query: str) -> List[Reminder]: data_retriever = DataRetriever(file_path=self.export_reminders()) results = data_retriever.forward(query=query) - + reminders = [] for row in results: reminder = Reminder.from_id(self.event_store, row['ID']) @@ -64,23 +67,23 @@ def text_query(self, text: str) -> List[Reminder]: """Perform a natural language query and return a list of Reminder objects.""" # Export reminders to a temporary CSV file csv_file = self.export_reminders() - + # Read the CSV file into a pandas DataFrame df = pd.read_csv(csv_file) - + # Get the DataFrame schema and data df_schema = df.columns.tolist() df_data = df.values.tolist() - + # Use DFSQLModule to convert natural language to SQL sql_query = dfsql_call(text=text, df_schema=df_schema, df_data=df_data) - + # Use the generated SQL query to fetch reminders reminders = self.query(sql_query) - + # Clean up the temporary CSV file os.unlink(csv_file) - + return reminders def export_reminders(self, filename=None, days=7) -> str: @@ -102,13 +105,14 @@ def create_reminder_from_generated(self, prompt: str, list_name: str) -> Reminde # Generate iCalendar data from the prompt ical_string = generate_i_calendar_call(prompt) - + # Create the reminder using the generated iCalendar data reminder = Reminder.create_from_rfc5545(self.event_store, ical_string, reminder_list.ek_calendar) return reminder + def main(): - app = ReminderApp() + app = RemindersApp() app.request_access() try: @@ -173,10 +177,11 @@ def main(): # Verify the list was removed print("Final lists:", app.get_all_lists()) + def main2(): from dspygen.utils.dspy_tools import init_dspy init_dspy() - app = ReminderApp() + app = RemindersApp() # Test natural language query # search_results = app.text_query("Find all reminders") @@ -185,6 +190,7 @@ def main2(): for reminder in search_results: print(reminder) + if __name__ == "__main__": main2() # main() diff --git a/src/dspygen/experiments/cal_apps/reminder_integration_testing.py b/src/dspygen/pyautomator/reminders/reminder_integration_testing.py similarity index 97% rename from src/dspygen/experiments/cal_apps/reminder_integration_testing.py rename to src/dspygen/pyautomator/reminders/reminder_integration_testing.py index 22b00c4..fb9f6a6 100644 --- a/src/dspygen/experiments/cal_apps/reminder_integration_testing.py +++ b/src/dspygen/pyautomator/reminders/reminder_integration_testing.py @@ -3,10 +3,12 @@ import inject -from dspygen.experiments.cal_apps.reminder import Reminder, ReminderError -from dspygen.experiments.cal_apps.alarm import Alarm + import EventKit +from dspygen.pyautomator.event_kit.alarm import Alarm +from dspygen.pyautomator.event_kit.reminder import Reminder, ReminderError + @inject.autoparams() def create_reminder(event_store:EventKit.EKEventStore, title: str, calendar: EventKit.EKCalendar, diff --git a/src/dspygen/experiments/cal_apps/wintermute_reminder.py b/src/dspygen/pyautomator/reminders/wintermute_reminder.py similarity index 92% rename from src/dspygen/experiments/cal_apps/wintermute_reminder.py rename to src/dspygen/pyautomator/reminders/wintermute_reminder.py index 1384bd8..6d21c87 100644 --- a/src/dspygen/experiments/cal_apps/wintermute_reminder.py +++ b/src/dspygen/pyautomator/reminders/wintermute_reminder.py @@ -1,5 +1,6 @@ import time -from dspygen.experiments.cal_apps.reminder import Reminder + +from dspygen.pyautomator.event_kit.reminder import Reminder from dspygen.utils.dspy_tools import init_dspy from dspygen.modules.comment_module import comment_call diff --git a/src/dspygen/pyautomator/safari/__init__.py b/src/dspygen/pyautomator/safari/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dspygen/pyautomator/safari/safari_app.py b/src/dspygen/pyautomator/safari/safari_app.py new file mode 100644 index 0000000..6f2c498 --- /dev/null +++ b/src/dspygen/pyautomator/safari/safari_app.py @@ -0,0 +1,143 @@ +import os +import requests +import json +import pyperclip +import time +from html2text import HTML2Text + +from dspygen.pyautomator.base_app import BaseApp + + +class SafariApp(BaseApp): + def __init__(self): + super().__init__("Safari") + + def get_current_tab(self): + script = """ + function getCurrentTab() { + const Safari = Application('Safari'); + const tab = Safari.windows[0].currentTab(); + return JSON.stringify({ + url: tab.url(), + name: tab.name() + }); + } + getCurrentTab(); + """ + result = self.execute_jxa(script) + return json.loads(result) + + def open_url(self, url): + script = f""" + const Safari = Application('Safari'); + const tab = Safari.windows[0].currentTab(); + tab.url = '{url}'; + """ + self.execute_jxa(script) + + def execute_js_in_tab(self, js_code): + script = f""" + function executeInTab() {{ + const Safari = Application('Safari'); + const tab = Safari.windows[0].currentTab(); + return Safari.doJavaScript(`{js_code}`, {{in: tab}}); + }} + executeInTab(); + """ + return self.execute_jxa(script) + + def get_page_content(self): + js_code = "document.body.innerHTML;" + return self.execute_js_in_tab(js_code) + + def get_page_links(self): + js_code = """ + JSON.stringify( + Array.from(document.querySelectorAll('a')) + .map(a => ({href: a.href, text: a.textContent.trim()})) + ); + """ + result = self.execute_js_in_tab(js_code) + return json.loads(result) + + def search_google(self, query): + self.open_url("https://www.google.com") + js_code = f""" + document.querySelector('textarea[name="q"]').value = "{query}"; + document.querySelector('form').submit(); + """ + self.execute_js_in_tab(js_code) + + def convert_to_markdown(self, output_file=None): + script = """ + (() => { + const Safari = Application('Safari'); + const tab = Safari.windows[0].currentTab(); + + const turndownURL = "https://unpkg.com/turndown/dist/turndown.js"; + const nsURL = $.NSURL.URLWithString($(turndownURL)); + const data = $.NSData.dataWithContentsOfURL(nsURL); + const turndownSrc = $.NSString.alloc.initWithDataEncoding(data, $.NSUTF8StringEncoding).js; + + const scriptSrc = turndownSrc + + ` + var turndownService = new TurndownService({ + headingStyle: 'atx', + codeBlockStyle: 'fenced', + bullet: '-' + }); + /* Ignore 'script' and 'nav' elements to keep + markdown cleaner */ + turndownService.remove('script'); + turndownService.remove('nav'); + turndownService.turndown(document.body); + `; + const result = Safari.doJavaScript(`${scriptSrc}`, {in: tab}); + + // Copy the result to clipboard + const app = Application.currentApplication(); + app.includeStandardAdditions = true; + app.setTheClipboardTo(result); + })() + """ + self.execute_jxa(script) + + # Give some time for the clipboard to be updated + time.sleep(0.5) + + # Get the result from the clipboard + markdown_content = pyperclip.paste() + + if output_file: + with open(output_file, 'w', encoding='utf-8') as f: + f.write(markdown_content) + print(f"Markdown content saved to: {output_file}") + + return markdown_content + +def main(): + safari = SafariApp() + safari.activate_app() + + # Example usage + current_tab = safari.get_current_tab() + print(f"Current tab: {current_tab['name']} - {current_tab['url']}") + + # safari.open_url("https://www.linkedin.com/in/seanchatman/") + content = safari.get_page_content() + print(content) + # print(f"Page content length: {len(content)}") + + # links = safari.get_page_links() + # print(f"Number of links on the page: {len(links)}") + + # safari.search_google("Python programming") + + # Convert to Markdown and save to file + output_file = "converted_page.md" + markdown = safari.convert_to_markdown(output_file=output_file) + print(f"Converted Markdown length: {len(markdown)}") + print(f"Markdown content saved to: {output_file}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/dspygen/rm/data_retriever.py b/src/dspygen/rm/data_retriever.py index 425cba1..b727124 100644 --- a/src/dspygen/rm/data_retriever.py +++ b/src/dspygen/rm/data_retriever.py @@ -73,6 +73,10 @@ def read_any(filepath, query, read_options=None): class DataRetriever(dspy.Retrieve): + supported_extensions = ['.csv', '.xls', '.xlsx', '.pickle', '.pkl', '.h5', '.hdf', + '.sql', '.db', '.json', '.parquet', '.orc', '.feather', + '.gbq', '.html', '.xml', '.stata', '.sas', '.sav', '.dta', '.fwf'] + def __init__(self, file_path: str | Path, query: str = "", return_columns=None, read_options=None, pipeline=None, step=None, **kwargs): super().__init__() @@ -91,6 +95,10 @@ def __init__(self, file_path: str | Path, query: str = "", return_columns=None, # Read the data using the read_any function self.df = read_any(file_path, query, read_options) # No SQL query here + @classmethod + def supports_file_type(cls, file_extension: str) -> bool: + return file_extension.lower() in cls.supported_extensions + def forward(self, query: str = None, k: int = None, **kwargs) -> list[dict]: # Check if a SQL query is provided if query: diff --git a/src/dspygen/rm/doc_retriever.py b/src/dspygen/rm/doc_retriever.py index 8e3d880..2b1af29 100644 --- a/src/dspygen/rm/doc_retriever.py +++ b/src/dspygen/rm/doc_retriever.py @@ -66,11 +66,16 @@ def read_any(file_path): class DocRetriever(dspy.Retrieve): + supported_extensions = ['.epub', '.pdf', '.txt', '.md', '.docx'] + def __init__(self, path, **kwargs): - # Read the file from any acceptable text file type super().__init__() self.path = path + @classmethod + def supports_file_type(cls, file_extension: str) -> bool: + return file_extension.lower() in cls.supported_extensions + def read_chunks(self, chunk_chars): text = read_any(self.path) return [text[i:i + chunk_chars] for i in range(0, len(text), chunk_chars)] diff --git a/src/dspygen/rm/retrievers.README.md b/src/dspygen/rm/retrievers.README.md new file mode 100644 index 0000000..e167af5 --- /dev/null +++ b/src/dspygen/rm/retrievers.README.md @@ -0,0 +1,159 @@ +# Advanced Document and Data Retrieval System + +This package provides a suite of high-performance retriever classes designed for efficient handling of various document and data file formats. Built with scalability and flexibility in mind, these retrievers integrate seamlessly with DSPy for advanced data processing pipelines. + +## Key Features + +- Multi-format support for documents and structured data +- Efficient text extraction and data querying +- Seamless integration with DSPy pipelines +- Optimized for large-scale data processing + +## Retriever Classes + +### ChatGPTChromaDBRetriever + +`ChatGPTChromaDBRetriever` is a sophisticated retriever class designed for efficient storage, indexing, and retrieval of ChatGPT conversation data. It leverages ChromaDB for vector storage and similarity search capabilities, making it ideal for large-scale conversation analysis and information retrieval tasks. + +#### Key Features + +- Efficient storage and indexing of ChatGPT conversations +- Vector-based similarity search using ChromaDB +- Automatic processing and updating of conversation data +- Customizable embedding functions (default: Ollama embedding) +- Seamless integration with DSPy pipelines + +#### Usage Example + +```python +from dspygen.rm.chatgpt_chromadb_retriever import ChatGPTChromaDBRetriever + +# Initialize the retriever +retriever = ChatGPTChromaDBRetriever( + json_file_path="path/to/conversations.json", + collection_name="chatgpt_conversations", + persist_directory="path/to/persist", + check_for_updates=True +) + +# Perform a simple query +query = "What are the key features of transformer models?" +results = retriever.forward(query, k=3) + +print("Top 3 relevant passages:") +for i, result in enumerate(results, 1): + print(f"{i}. {result[:200]}...") # Print first 200 characters of each result + +# Use in a DSPy pipeline +class ConversationAnalysisPipeline(dspy.Module): + def __init__(self): + self.retriever = ChatGPTChromaDBRetriever() + + def forward(self, query): + relevant_passages = self.retriever.forward(query, k=5) + return dspy.Prediction(relevant_info=relevant_passages) + +pipeline = ConversationAnalysisPipeline() +result = pipeline("Explain the concept of attention in neural networks") +print("Relevant information:", result.relevant_info) + +# Advanced usage with filters +assistant_responses = retriever.forward( + query="Explain the benefits of transfer learning", + k=3, + role="assistant" # Only retrieve responses from the assistant +) + +print("Top 3 assistant explanations on transfer learning:") +for i, response in enumerate(assistant_responses, 1): + print(f"{i}. {response[:200]}...") # Print first 200 characters of each response +``` + + + +### DataRetriever + +`DataRetriever` is a versatile and powerful class designed for efficient retrieval and querying of structured data from various file formats. It seamlessly integrates with DSPy pipelines and supports SQL-like querying capabilities. + +#### Key Features + +- Supports multiple file formats including CSV, Excel, SQL databases, JSON, Parquet, and more +- SQL-like querying capabilities using pandasql +- Efficient data loading and processing +- Seamless integration with DSPy pipelines + +#### Usage Example + +```python +from dspygen.rm.data_retriever import DataRetriever + +# Initialize DataRetriever with a CSV file +data_retriever = DataRetriever(file_path='sales_data.csv') + +# Perform a SQL-like query on the data +query = "SELECT product_name, SUM(sales_amount) as total_sales FROM df GROUP BY product_name ORDER BY total_sales DESC LIMIT 5" +top_products = data_retriever.forward(query=query) + +print("Top 5 Products by Sales:") +for product in top_products: + print(f"{product['product_name']}: ${product['total_sales']:.2f}") + +# Use in a DSPy pipeline +class SalesAnalysisPipeline(dspy.Module): + def __init__(self): + self.retriever = DataRetriever(file_path='sales_data.csv') + + def forward(self, query): + results = self.retriever.forward(query=query) + return dspy.Prediction(analysis_results=results) + +pipeline = SalesAnalysisPipeline() +result = pipeline("SELECT region, AVG(sales_amount) as avg_sales FROM df GROUP BY region") +print("Average Sales by Region:", result.analysis_results) +``` + +### DocRetriever + +`DocRetriever` is an advanced text extraction tool designed for multiple document formats. It supports reading and cleaning text from various document types, making it ideal for preprocessing text data for NLP tasks. + +#### Usage Example + + +```python +from dspygen.rm.doc_retriever import DocRetriever + +# Initialize DocRetriever with a PDF file +doc_retriever = DocRetriever(path='example_document.pdf') + +# Extract full text from the document +full_text = doc_retriever.forward() +print("Full text length:", len(full_text)) + +# Extract text in chunks of 1000 characters +text_chunks = doc_retriever.forward(chunk_chars=1000) +print("Number of chunks:", len(text_chunks)) +print("First chunk preview:", text_chunks[0][:100]) + +# Use in a DSPy pipeline +class DocumentAnalysisPipeline(dspy.Module): + def __init__(self, document_path): + self.retriever = DocRetriever(path=document_path) + + def forward(self, chunk_size=None): + if chunk_size: + return dspy.Prediction(document_chunks=self.retriever.forward(chunk_chars=chunk_size)) + return dspy.Prediction(full_text=self.retriever.forward()) + +pipeline = DocumentAnalysisPipeline('example_document.pdf') +result = pipeline(chunk_size=500) +print("Number of document chunks:", len(result.document_chunks)) +``` + +#### Supported File Types + +- EPUB (.epub) +- PDF (.pdf) +- Text (.txt) +- Markdown (.md) +- Word Document (.docx) + diff --git a/src/dspygen/subcommands/wkf_cmd.py b/src/dspygen/subcommands/wkf_cmd.py index 9028565..5c054f3 100644 --- a/src/dspygen/subcommands/wkf_cmd.py +++ b/src/dspygen/subcommands/wkf_cmd.py @@ -1,11 +1,52 @@ import typer - +import os +from pathlib import Path +import yaml +from datetime import datetime, timedelta +from apscheduler.schedulers.background import BackgroundScheduler from dspygen.utils.cli_tools import chatbot -from dspygen.workflow.workflow_executor import execute_workflow -from dspygen.workflow.workflow_models import Workflow +from dspygen.workflow.workflow_executor import execute_workflow, schedule_workflow +from dspygen.workflow.workflow_models import Workflow, Job +from sungen.typetemp.functional import render +from dspygen.utils.file_tools import rm_dir app = typer.Typer(help="Language Workflow Domain Specific Language commands for DSPyGen.") +# Workflow template +workflow_template = """ +name: {{ name }} +triggers: + - cron: "0 0 * * *" # Daily at midnight + +jobs: + - name: ExampleJob + runner: python + steps: + - name: ExampleStep + code: | + print("Hello from {{ name }} workflow!") + +""" + +def wf_dir(): + """Returns the directory where workflows are stored.""" + return os.path.join(os.path.dirname(__file__), "..", "workflows") + +@app.command("new") +def new_workflow(name: str = typer.Argument(...)): + """Generates a new workflow YAML file.""" + to = wf_dir() + os.makedirs(to, exist_ok=True) + file_path = os.path.join(to, f"{name.lower()}_workflow.yaml") + + content = render(workflow_template, name=name) + + with open(file_path, 'w') as f: + f.write(content) + + typer.echo(f"New workflow created at: {file_path}") + typer.echo("Workflow content:") + typer.echo(content) @app.command("run") def run_workflow(yaml_file: str = typer.Argument("/Users/sac/dev/dspygen/src/dspygen/experiments/workflow/control_flow_workflow.yaml")): @@ -14,8 +55,65 @@ def run_workflow(yaml_file: str = typer.Argument("/Users/sac/dev/dspygen/src/dsp """ wf = Workflow.from_yaml(yaml_file) result = execute_workflow(wf) - # print(result) + typer.echo(f"Workflow execution result: {result}") +def run_workflows_in_directory(directory: str, recursive: bool = True): + """ + Run all YAML workflow files in the specified directory and its subdirectories. + """ + scheduler = BackgroundScheduler() + scheduler.start() + + workflows_found = False + + def process_workflow_file(file_path): + nonlocal workflows_found + try: + workflow = Workflow.from_yaml(file_path) + typer.echo(f"Scheduling workflow from file: {file_path}") + schedule_workflow(workflow, scheduler) + workflows_found = True + except Exception as e: + typer.echo(f"Error processing workflow file {file_path}: {str(e)}", err=True) + + def search_workflows(path): + for root, _, files in os.walk(path): + for file in files: + if file.endswith(('.yaml', '.yml')): + file_path = os.path.join(root, file) + process_workflow_file(file_path) + if not recursive: + break # Stop after processing the top-level directory + + search_workflows(directory) + + if not workflows_found: + typer.echo("No workflows found or scheduled.") + scheduler.shutdown() + return None + + return scheduler + +@app.command("run-all") +def run_all_workflows( + directory: str = typer.Argument(".", help="Directory containing YAML workflow files"), + recursive: bool = typer.Option(True, help="Search subdirectories recursively") +): + """ + Run all YAML workflow files in the specified directory and its subdirectories. + """ + scheduler = run_workflows_in_directory(directory, recursive) + if scheduler: + typer.echo("All workflows scheduled. Press Ctrl+C to exit.") + try: + # Keep the script running + scheduler.print_jobs() + while True: + pass + except (KeyboardInterrupt, SystemExit): + typer.echo("Shutting down scheduler...") + scheduler.shutdown() + typer.echo("Scheduler shut down. Exiting.") TUTOR_CONTEXT = '''The DSPyGen DSL has several key elements that you'll need to grasp: @@ -183,11 +281,188 @@ class WorkflowDSLModel(BaseModel, YAMLMixin): ''' - @app.command(name="tutor") def tutor(question: str = ""): """Guide you through developing a project with DSPyGen DSL.""" from dspygen.utils.dspy_tools import init_dspy init_dspy(max_tokens=3000, model="gpt-4") - chatbot(question, TUTOR_CONTEXT) \ No newline at end of file + chatbot(question, TUTOR_CONTEXT) + +@app.command("list") +def list_workflows(directory: str = typer.Argument(wf_dir(), help="Directory containing workflow files")): + """List all workflows in a specified directory.""" + workflows = [] + for file in os.listdir(directory): + if file.endswith(('.yaml', '.yml')): + workflows.append(file) + + if workflows: + typer.echo("Available workflows:") + for wf in workflows: + typer.echo(f"- {wf}") + else: + typer.echo("No workflows found in the specified directory.") + +@app.command("show") +def show_workflow(workflow_name: str): + """Display details about a specific workflow.""" + file_path = os.path.join(wf_dir(), f"{workflow_name}.yaml") + if not os.path.exists(file_path): + typer.echo(f"Workflow {workflow_name} not found.") + return + + with open(file_path, 'r') as f: + content = f.read() + + typer.echo(f"Content of workflow {workflow_name}:") + typer.echo(content) + +@app.command("delete") +def delete_workflow(workflow_name: str): + """Delete a specific workflow file.""" + file_path = os.path.join(wf_dir(), f"{workflow_name}.yaml") + if not os.path.exists(file_path): + typer.echo(f"Workflow {workflow_name} not found.") + return + + os.remove(file_path) + typer.echo(f"Workflow {workflow_name} has been deleted.") + +@app.command("trigger") +def trigger_workflow(workflow_name: str): + """Manually trigger a specific workflow.""" + file_path = os.path.join(wf_dir(), f"{workflow_name}.yaml") + if not os.path.exists(file_path): + typer.echo(f"Workflow {workflow_name} not found.") + return + + wf = Workflow.from_yaml(file_path) + result = execute_workflow(wf) + typer.echo(f"Workflow {workflow_name} triggered. Execution result: {result}") + +@app.command("pause") +def pause_workflow(workflow_name: str): + """Pause a specific workflow.""" + file_path = os.path.join(wf_dir(), f"{workflow_name}.yaml") + if not os.path.exists(file_path): + typer.echo(f"Workflow {workflow_name} not found.") + return + + wf = Workflow.from_yaml(file_path) + wf.paused = True + wf.to_yaml(file_path) + typer.echo(f"Workflow {workflow_name} has been paused.") + +@app.command("unpause") +def unpause_workflow(workflow_name: str): + """Unpause a specific workflow.""" + file_path = os.path.join(wf_dir(), f"{workflow_name}.yaml") + if not os.path.exists(file_path): + typer.echo(f"Workflow {workflow_name} not found.") + return + + wf = Workflow.from_yaml(file_path) + wf.paused = False + wf.to_yaml(file_path) + typer.echo(f"Workflow {workflow_name} has been unpaused.") + +@app.command("test") +def test_workflow(workflow_name: str): + """Test a workflow without executing its tasks.""" + file_path = os.path.join(wf_dir(), f"{workflow_name}.yaml") + if not os.path.exists(file_path): + typer.echo(f"Workflow {workflow_name} not found.") + return + + wf = Workflow.from_yaml(file_path) + typer.echo(f"Testing workflow: {workflow_name}") + typer.echo(f"Number of jobs: {len(wf.jobs)}") + for job in wf.jobs: + typer.echo(f" Job: {job.name}") + typer.echo(f" Number of steps: {len(job.steps)}") + typer.echo("Workflow structure is valid.") + +@app.command("list-jobs") +def list_jobs(workflow_name: str): + """List all jobs in a specific workflow.""" + file_path = os.path.join(wf_dir(), f"{workflow_name}.yaml") + if not os.path.exists(file_path): + typer.echo(f"Workflow {workflow_name} not found.") + return + + wf = Workflow.from_yaml(file_path) + typer.echo(f"Jobs in workflow {workflow_name}:") + for job in wf.jobs: + typer.echo(f"- {job.name}") + +@app.command("test-job") +def test_job(workflow_name: str, job_name: str): + """Test a specific job in a workflow.""" + file_path = os.path.join(wf_dir(), f"{workflow_name}.yaml") + if not os.path.exists(file_path): + typer.echo(f"Workflow {workflow_name} not found.") + return + + wf = Workflow.from_yaml(file_path) + job = next((job for job in wf.jobs if job.name == job_name), None) + if not job: + typer.echo(f"Job {job_name} not found in workflow {workflow_name}.") + return + + typer.echo(f"Testing job: {job_name}") + typer.echo(f"Number of steps: {len(job.steps)}") + for step in job.steps: + typer.echo(f" Step: {step.name}") + typer.echo(f" Code: {step.code[:50]}...") # Show first 50 characters of code + typer.echo("Job structure is valid.") + +@app.command("clear-job") +def clear_job(workflow_name: str, job_name: str): + """Clear the state of a specific job instance.""" + file_path = os.path.join(wf_dir(), f"{workflow_name}.yaml") + if not os.path.exists(file_path): + typer.echo(f"Workflow {workflow_name} not found.") + return + + wf = Workflow.from_yaml(file_path) + job = next((job for job in wf.jobs if job.name == job_name), None) + if not job: + typer.echo(f"Job {job_name} not found in workflow {workflow_name}.") + return + + # Implement job state clearing logic here + typer.echo(f"Clearing state for job {job_name} in workflow {workflow_name}... (Not implemented)") + +@app.command("backfill") +def backfill(workflow_name: str, start_date: str, end_date: str): + """Run a workflow for a specified historical time range.""" + typer.echo(f"Backfilling workflow {workflow_name} from {start_date} to {end_date}... (Not implemented)") + +@app.command("logs") +def show_logs(workflow_name: str, job_name: str): + """Display logs for a specific job run.""" + typer.echo(f"Showing logs for job {job_name} in workflow {workflow_name}... (Not implemented)") + +@app.command("export") +def export_workflow(workflow_name: str, output_file: str): + """Export a workflow definition to a file.""" + typer.echo(f"Exporting workflow {workflow_name} to {output_file}... (Not implemented)") + +@app.command("import") +def import_workflow(input_file: str): + """Import a workflow definition from a file.""" + typer.echo(f"Importing workflow from {input_file}... (Not implemented)") + +@app.command("scheduler-health") +def scheduler_health(): + """Check the status of the scheduler.""" + typer.echo("Checking scheduler health... (Not implemented)") + +@app.command("version") +def version(): + """Display the version of the workflow system.""" + typer.echo("Workflow system version: X.Y.Z (Not implemented)") + +if __name__ == "__main__": + app() \ No newline at end of file diff --git a/src/dspygen/workflow/data_analysis_workflow.yaml b/src/dspygen/workflow/data_analysis_workflow.yaml index 86f47f4..00cfa6a 100644 --- a/src/dspygen/workflow/data_analysis_workflow.yaml +++ b/src/dspygen/workflow/data_analysis_workflow.yaml @@ -1,5 +1,7 @@ name: DataAnalysisWorkflow -triggers: manual +triggers: + - cron: "0 0 * * *" # Run daily at midnight + - cron: "0 */4 * * *" # Run every 4 hours imports: - /Users/sac/dev/dspygen/src/dspygen/workflow/data_preparation_workflow.yaml jobs: @@ -24,6 +26,4 @@ jobs: from dspygen.dsl.dsl_pipeline_executor import execute_pipeline context = execute_pipeline('/Users/sac/dev/dspygen/tests/pipeline/data_hello_world_pipeline.yaml', init_ctx={"csv_file": "/Users/sac/dev/dspygen/tests/data/greek.csv"}) print(context) - - env: {} diff --git a/src/dspygen/workflow/workflow_executor.py b/src/dspygen/workflow/workflow_executor.py index 93f82d2..631840b 100644 --- a/src/dspygen/workflow/workflow_executor.py +++ b/src/dspygen/workflow/workflow_executor.py @@ -1,18 +1,38 @@ import copy from typing import Optional, Dict, Any - from sungen.typetemp.functional import render, render_native -from dspygen.workflow.workflow_models import Workflow, Action, Job +from dspygen.workflow.workflow_models import Workflow, Action, Job, DateTrigger, CronTrigger from loguru import logger - +from apscheduler.schedulers.base import BaseScheduler +from apscheduler.triggers.cron import CronTrigger as APSchedulerCronTrigger +from apscheduler.triggers.date import DateTrigger as APSchedulerDateTrigger +from datetime import datetime +import pytz +import sys + +# Configure logger with timestamp and log level +logger.remove() # Remove default handler +logger.add( + sys.stderr, + format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}", + level="DEBUG" +) +logger.add( + "workflow_executor.log", + format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}", + level="DEBUG", + rotation="1 MB" +) def initialize_context(init_ctx: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """Initializes the workflow context.""" + logger.debug(f"Initializing context with: {init_ctx}") return copy.deepcopy(init_ctx) if init_ctx else {} def update_context(context: Dict[str, Any], updates: Dict[str, Any]) -> Dict[str, Any]: """Updates the workflow context with new values.""" + # logger.debug(f"Updating context. Current: {context}, Updates: {updates}") # Create a copy of context with only python primitives new_context = {k: v for k, v in context.items() if isinstance(v, (int, float, str, bool, list, dict))} @@ -37,14 +57,18 @@ def update_context(context: Dict[str, Any], updates: Dict[str, Any]) -> Dict[str else: # Non-string values are added to the context unchanged rendered_context[arg] = value + # logger.debug(f"Updated context: {rendered_context}") return rendered_context def evaluate_condition(condition: str, context: Dict[str, Any]) -> bool: """Evaluates a condition within the current context.""" + logger.debug(f"Evaluating condition: '{condition}' with context: {context}") try: safe_context = copy.deepcopy(context) - return eval(condition, {}, safe_context) + result = eval(condition, {}, safe_context) + logger.debug(f"Condition result: {result}") + return result except Exception as e: logger.error(f"Error evaluating condition '{condition}': {e}") return False @@ -56,8 +80,14 @@ def execute_job(job: Job, context: Dict[str, Any]) -> Dict[str, Any]: job_context = update_context(context, {}) # Isolate context for the job for action in job.steps: - job_context = execute_action(action, job_context) # Execute each action - + logger.info(f"Executing action: {action.name}") + try: + job_context = execute_action(action, job_context) # Execute each action + logger.info(f"Finished executing action: {action.name}") + except Exception as e: + logger.error(f"Error executing action {action.name}: {str(e)}") + + logger.debug(f"Job {job.name} completed. Updated context: {job_context}") return job_context @@ -70,14 +100,20 @@ def execute_action(action: Action, context: Dict[str, Any]) -> Dict[str, Any]: logger.info(f"Condition for action '{action.name}' not met, skipping.") return context # Skip the action if condition not met - action_context = update_context(context, {})# Isolate context for the action + action_context = update_context(context, {}) # Isolate context for the action if action.code: + logger.debug(f"Executing code for action '{action.name}'") rendered_code = render(action.code, **action_context) - # Execute action's code, allowing it to modify the action-specific context - exec(rendered_code, action_context, action_context) + logger.debug(f"Rendered code: {rendered_code}") + try: + exec(rendered_code, action_context, action_context) + logger.info(f"Code execution for action '{action.name}' completed successfully") + except Exception as e: + logger.error(f"Error executing code for action '{action.name}': {str(e)}") context = update_context(context, action_context) # Update global context with changes + logger.debug(f"Action '{action.name}' completed. Updated context: {context}") return context @@ -90,9 +126,54 @@ def execute_workflow(workflow: Workflow, init_ctx: Optional[Dict[str, Any]] = No workflow.topological_sort() for job in workflow.jobs: - global_context = execute_job(job, global_context) # Execute each job + logger.info(f"Starting execution of job: {job.name}") + try: + global_context = execute_job(job, global_context) # Execute each job + logger.info(f"Finished execution of job: {job.name}") + except Exception as e: + logger.error(f"Error executing job {job.name}: {str(e)}") if '__builtins__' in global_context: del global_context['__builtins__'] # Remove builtins from context + logger.info(f"Workflow '{workflow.name}' completed. Final context: {global_context}") return global_context + + +def schedule_workflow(workflow: Workflow, scheduler: BaseScheduler): + """Schedules a workflow using the provided scheduler.""" + logger.info(f"Scheduling workflow: {workflow.name}") + for trigger in workflow.triggers: + if isinstance(trigger, CronTrigger): + logger.debug(f"Adding cron job for trigger: {trigger.cron}") + job = scheduler.add_job( + execute_workflow, + APSchedulerCronTrigger.from_crontab(trigger.cron, timezone=pytz.UTC), + args=[workflow], + timezone=pytz.UTC + ) + logger.debug(f"Job added: {str(job)}") + else: + logger.error(f"Unknown trigger type: {type(trigger)}") + + logger.info(f"Workflow '{workflow.name}' scheduled successfully") + logger.debug(f"All jobs: {scheduler.get_jobs()}") + return scheduler + + +if __name__ == "__main__": + from apscheduler.schedulers.background import BackgroundScheduler + + workflow = Workflow.from_yaml("path/to/your/workflow.yaml") + scheduler = BackgroundScheduler() + scheduler.start() + + schedule_workflow(workflow, scheduler) + + try: + # Keep the script running + while True: + pass + except (KeyboardInterrupt, SystemExit): + scheduler.shutdown() + diff --git a/src/dspygen/workflow/workflow_models.py b/src/dspygen/workflow/workflow_models.py index 12e0589..a99329c 100644 --- a/src/dspygen/workflow/workflow_models.py +++ b/src/dspygen/workflow/workflow_models.py @@ -23,6 +23,7 @@ from typing import List, Union, Dict, Any, Optional from pydantic import BaseModel, Field +from datetime import datetime from dspygen.utils.yaml_tools import YAMLMixin @@ -116,10 +117,15 @@ class Job(BaseModel): sla_seconds: Optional[int] = Field(None, description="Service Level Agreement for the job completion, specified in seconds.") -# class Trigger(BaseModel): -# schedule: Optional[str] = Field(None, description="Cron-like schedule for automated workflow triggering.") -# webhook: Optional[str] = Field(None, description="Webhook URL for external triggering of the workflow.") +class CronTrigger(BaseModel): + type: str = Field("cron") + cron: str = Field(..., description="Cron-like schedule for workflow triggering") +class DateTrigger(BaseModel): + type: str = Field("date") + run_date: Union[str, datetime] = Field(..., description="Date and time to run the workflow, or 'now' for immediate execution") + +Trigger = Union[CronTrigger, DateTrigger] class Workflow(BaseModel, YAMLMixin): """ @@ -132,8 +138,7 @@ class Workflow(BaseModel, YAMLMixin): """ name: str = Field(..., description="The unique name of the workflow.") description: Optional[str] = Field(None, description="A brief description of the workflow.") - # triggers: Union[Trigger, List[Trigger]] = Field(..., description="Events that trigger the workflow execution.") - triggers: Optional[Union[str, List[str]]] = Field([], description="Events that trigger the workflow execution.") + triggers: List[Union[CronTrigger, DateTrigger]] = Field([], description="List of triggers for the workflow execution.") jobs: List[Job] = Field( ..., description="A collection of jobs that are defined within the workflow." ) diff --git a/src/dspygen/writer/Tetris_Blog_Phi3Med.md b/src/dspygen/writer/Tetris_Blog_Phi3Med.md new file mode 100644 index 0000000..2fcb536 --- /dev/null +++ b/src/dspygen/writer/Tetris_Blog_Phi3Med.md @@ -0,0 +1,6 @@ +# Book List + +- The Great Gatsby +- 1984 +- Brave New World +- The Catcher in the Rye \ No newline at end of file diff --git a/src/dspygen/writer/data_writer.py b/src/dspygen/writer/data_writer.py index 26d248b..987e3fa 100644 --- a/src/dspygen/writer/data_writer.py +++ b/src/dspygen/writer/data_writer.py @@ -17,10 +17,25 @@ def __init__(self, data, file_path="", write_options=None): # Handle different data formats if file_extension == '.csv': - if isinstance(data, dict) and all(isinstance(v, list) for v in data.values()): + if isinstance(data, list) and all(isinstance(item, dict) for item in data): self.df = pd.DataFrame(data) + elif isinstance(data, dict) and all(isinstance(v, list) for v in data.values()): + self.df = pd.DataFrame(data) + else: + raise ValueError("For CSV files, data must be a list of dictionaries or a dictionary of lists.") + + # Check for any variation of 'ID' column + id_column = next((col for col in self.df.columns if col.lower() == 'id'), None) + + # If no ID column exists, add it + if id_column is None: + self.df.insert(0, 'ID', range(len(self.df))) else: - raise ValueError("For CSV files, data must be a dictionary of lists.") + # If ID column exists but is not the first column, move it to the front + if self.df.columns.get_loc(id_column) != 0: + cols = self.df.columns.tolist() + cols.insert(0, cols.pop(cols.index(id_column))) + self.df = self.df[cols] elif file_extension == '.md': if isinstance(data, str): self.md_content = data @@ -46,19 +61,12 @@ def forward(self, **kwargs): file_extension = file_extension.lower() if file_extension == '.csv': - write_functions = { - '.csv': self.df.to_csv, - # Add more mappings for different file types - } print("write " + self.file_path) - if file_extension in write_functions: - write_function = write_functions[file_extension] - try: - write_function(self.file_path, **self.write_options) - except Exception as e: - raise ValueError(f"Failed to write to {self.file_path} due to: {e}") - else: - raise ValueError(f"Unsupported file type: {file_extension}") + try: + # Set index=False to prevent writing the index as a separate column + self.df.to_csv(self.file_path, index=False, **self.write_options) + except Exception as e: + raise ValueError(f"Failed to write to {self.file_path} due to: {e}") elif file_extension == '.md': print("write " + self.file_path) @@ -103,19 +111,43 @@ class FileNameModel(BaseModel): extension: str = Field("csv", description="File extension for the output file.") def main(): - # Example Usage for CSV - data_csv = { - 'Book Title': ['The Great Gatsby', '1984', 'Brave New World', 'The Catcher in the Rye'], - 'Author': ['F. Scott Fitzgerald', 'George Orwell', 'Aldous Huxley', 'J.D. Salinger'], - 'Price': [10.99, 9.99, 8.99, 12.99], - 'Sold Copies': [500, 800, 650, 450] - } - writer_csv = DataWriter(file_path="./data/Book_Title_Author_Price_Sold_Copies.csv", data=data_csv) - writer_csv.forward() - - # Example Usage for Markdown + # Test case 1: No ID column (should add 'ID' column) + data_csv1 = [ + {'Book Title': 'The Great Gatsby', 'Author': 'F. Scott Fitzgerald', 'Price': 10.99, 'Sold Copies': 500}, + {'Book Title': '1984', 'Author': 'George Orwell', 'Price': 9.99, 'Sold Copies': 800}, + {'Book Title': 'Brave New World', 'Author': 'Aldous Huxley', 'Price': 8.99, 'Sold Copies': 650}, + {'Book Title': 'The Catcher in the Rye', 'Author': 'J.D. Salinger', 'Price': 12.99, 'Sold Copies': 450} + ] + writer_csv1 = DataWriter(file_path="test1_no_id.csv", data=data_csv1) + writer_csv1.forward() + + # Test case 2: 'id' column exists (should keep existing 'id' and move to front) + data_csv2 = [ + {'id': 'A1', 'Book Title': 'To Kill a Mockingbird', 'Author': 'Harper Lee', 'Price': 11.99, 'Sold Copies': 750}, + {'id': 'B2', 'Book Title': 'Pride and Prejudice', 'Author': 'Jane Austen', 'Price': 7.99, 'Sold Copies': 950}, + ] + writer_csv2 = DataWriter(file_path="test2_existing_id.csv", data=data_csv2) + writer_csv2.forward() + + # Test case 3: 'ID' column exists but not at the front (should move to front) + data_csv3 = [ + {'Book Title': 'The Hobbit', 'ID': 'C3', 'Author': 'J.R.R. Tolkien', 'Price': 14.99, 'Sold Copies': 1000}, + {'Book Title': 'Dune', 'ID': 'D4', 'Author': 'Frank Herbert', 'Price': 13.99, 'Sold Copies': 850}, + ] + writer_csv3 = DataWriter(file_path="test3_id_not_front.csv", data=data_csv3) + writer_csv3.forward() + + # Test case 4: 'Id' column exists (should keep existing 'Id' and move to front) + data_csv4 = [ + {'Book Title': 'The Alchemist', 'Author': 'Paulo Coelho', 'Id': 'E5', 'Price': 9.99, 'Sold Copies': 1200}, + {'Book Title': 'The Da Vinci Code', 'Author': 'Dan Brown', 'Id': 'F6', 'Price': 12.99, 'Sold Copies': 1100}, + ] + writer_csv4 = DataWriter(file_path="test4_Id_exists.csv", data=data_csv4) + writer_csv4.forward() + + # Example Usage for Markdown (unchanged) data_md = "# Book List\n\n- The Great Gatsby\n- 1984\n- Brave New World\n- The Catcher in the Rye" - writer_md = DataWriter(file_path="./data/Tetris_Blog_Phi3Med.md", data=data_md) + writer_md = DataWriter(file_path="Tetris_Blog_Phi3Med.md", data=data_md) writer_md.forward() if __name__ == "__main__": diff --git a/tests/actor/test_actor.py b/tests/actor/test_actor.py index 44a1c79..626a186 100644 --- a/tests/actor/test_actor.py +++ b/tests/actor/test_actor.py @@ -1,33 +1,33 @@ -import asyncio - -import pytest - -from dspygen.rdddy.base_inhabitant import BaseInhabitant -from dspygen.rdddy.base_query import BaseQuery -from dspygen.rdddy.service_colony import ServiceColony - - -@pytest.fixture() -def service_colony(event_loop): - # Provide the event loop to the inhabitant system - return ServiceColony(event_loop) - - -class DummyInhabitant(BaseInhabitant): - def __init__(self, service_colony, inhabitant_id=None): - super().__init__(service_colony, inhabitant_id) - self.processed_query = None - - async def handle_query(self, query: BaseQuery): - self.processed_query = query - - -@pytest.mark.asyncio() -async def test_handler(service_colony): - inhabitant = await service_colony.inhabitant_of(DummyInhabitant) - - query = BaseQuery(content="Query1") - - await asyncio.sleep(0) - - await service_colony.publish(query) +# import asyncio +# +# import pytest +# +# from dspygen.rdddy.base_inhabitant import BaseInhabitant +# from dspygen.rdddy.base_query import BaseQuery +# from dspygen.rdddy.service_colony import ServiceColony +# +# +# @pytest.fixture() +# def service_colony(event_loop): +# # Provide the event loop to the inhabitant system +# return ServiceColony(event_loop) +# +# +# class DummyInhabitant(BaseInhabitant): +# def __init__(self, service_colony, inhabitant_id=None): +# super().__init__(service_colony, inhabitant_id) +# self.processed_query = None +# +# async def handle_query(self, query: BaseQuery): +# self.processed_query = query +# +# +# @pytest.mark.asyncio() +# async def test_handler(service_colony): +# inhabitant = await service_colony.inhabitant_of(DummyInhabitant) +# +# query = BaseQuery(content="Query1") +# +# await asyncio.sleep(0) +# +# await service_colony.publish(query) diff --git a/tests/experiments/cal_apps/test_reminder_app_bdd.py b/tests/experiments/cal_apps/test_reminder_app_bdd.py index 76e1bac..f4e5786 100644 --- a/tests/experiments/cal_apps/test_reminder_app_bdd.py +++ b/tests/experiments/cal_apps/test_reminder_app_bdd.py @@ -1,13 +1,14 @@ +import EventKit import pytest from pytest_bdd import scenario, given, when, then -from dspygen.experiments.cal_apps.reminder_app import ReminderApp -from dspygen.experiments.cal_apps.reminder_list import ReminderList -from dspygen.experiments.cal_apps.reminder import Reminder +from dspygen.pyautomator.reminders.reminder_app import RemindersApp +from dspygen.pyautomator.event_kit.reminder_list import ReminderList +from dspygen.pyautomator.event_kit.reminder import Reminder @pytest.fixture def reminder_app(): - app = ReminderApp() + app = RemindersApp() app.event_store = MockEventStore() # Mock the EventKit.EKCalendar class EventKit.EKCalendar = MockCalendar @@ -78,7 +79,7 @@ def test_clear_completed_reminders(): @given('the Reminder App is initialized') def reminder_app_initialized(reminder_app): - assert isinstance(reminder_app, ReminderApp) + assert isinstance(reminder_app, RemindersApp) @when('I add a new reminder list called "{list_name}"') diff --git a/tests/experiments/test_reminders_models.py b/tests/experiments/test_reminders_models.py deleted file mode 100644 index 0519ecb..0000000 --- a/tests/experiments/test_reminders_models.py +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/src/dspygen/experiments/cal_apps/recurrence_handler.py b/tests/test_advanced_integration_job_search_workflow.py similarity index 100% rename from src/dspygen/experiments/cal_apps/recurrence_handler.py rename to tests/test_advanced_integration_job_search_workflow.py diff --git a/tests/test_create_row_integration.py b/tests/test_create_row_integration.py new file mode 100644 index 0000000..1e06894 --- /dev/null +++ b/tests/test_create_row_integration.py @@ -0,0 +1,77 @@ +import pytest +import os +from dspygen.modules.create_row_module import create_row_call +from dspygen.utils.dspy_tools import init_dspy +from dspygen.pyautomator.reminders.reminder_app import RemindersApp +from dspygen.rm.data_retriever import DataRetriever + +@pytest.fixture +def setup_dspy(): + init_dspy() + +@pytest.fixture +def reminders_app(): + app = RemindersApp() + app.request_access() + return app + +@pytest.fixture +def csv_file(reminders_app): + csv_path = reminders_app.export_reminders() + yield csv_path + os.remove(csv_path) # Clean up the file after the test + +def test_create_row_integration(setup_dspy, reminders_app, csv_file): + # Load the initial data + data_retriever = DataRetriever(file_path=csv_file) + initial_data = data_retriever.forward() + + # Prepare the request + request = "Add a new task to buy groceries for the weekend, due on 2024-09-05, with high priority to Today" + + # Call the create_row_call function + updated_data = create_row_call(data=initial_data, request=request) + + # Assertions + assert len(updated_data) == len(initial_data) + 1 + new_row = updated_data[-1] + assert new_row['Title'] == 'Buy groceries for the weekend' + assert new_row['DueDate'] == '2024-09-05' + assert 'ID' in new_row # Ensure an ID is generated + assert new_row['Calendar'] == 'Today' # Assuming new tasks are added to 'Today' calendar + + +def test_create_row_with_notes(setup_dspy, reminders_app, csv_file): + data_retriever = DataRetriever(file_path=csv_file) + initial_data = data_retriever.forward() + + request = "Add a task to call Alice about the project meeting, due tomorrow, medium priority, and add a note to prepare agenda items" + + updated_data = create_row_call(data=initial_data, request=request) + + new_row = updated_data[-1] + assert new_row['Title'] == 'Call Alice about the project meeting' + assert 'prepare agenda items' in new_row['Notes'].lower() + + # Verify the new reminder is added to the app + new_reminders = reminders_app.text_query("Find reminders about calling Alice") + assert len(new_reminders) > 0 + assert any(r.title == 'Call Alice about the project meeting' for r in new_reminders) + +def test_create_row_with_existing_calendar(setup_dspy, reminders_app, csv_file): + data_retriever = DataRetriever(file_path=csv_file) + initial_data = data_retriever.forward() + + request = "Add a new health task to drink more water, due in 3 days, low priority" + + updated_data = create_row_call(data=initial_data, request=request) + + new_row = updated_data[-1] + assert new_row['Title'] == 'Drink more water' + assert new_row['Calendar'] == 'Health & Wellness' + assert str(new_row['Priority']) == '0' # Convert to string for comparison + + # Verify the new reminder is added to the app + new_reminders = reminders_app.text_query("Find reminders about drinking water in Health & Wellness calendar") + assert len(new_reminders) > 0 + assert any(r.title == 'Drink more water' for r in new_reminders) diff --git a/tests/test_data_retriever_stress.py b/tests/test_data_retriever_stress.py new file mode 100644 index 0000000..d4e75b7 --- /dev/null +++ b/tests/test_data_retriever_stress.py @@ -0,0 +1,83 @@ +import pytest +import time +from dspygen.rm.data_retriever import DataRetriever + +CSV_PATH = "/Users/sac/dev/dspygen/data/21KLinkedInConnections.csv" + +@pytest.fixture +def data_retriever(): + return DataRetriever(file_path=CSV_PATH) + +def test_large_dataset_loading(data_retriever): + start_time = time.time() + result = data_retriever.forward() + end_time = time.time() + + assert len(result) > 20000, "Dataset should contain over 20,000 records" + assert end_time - start_time < 5, "Loading should take less than 5 seconds" + +def test_complex_query_execution(data_retriever): + query = """ + SELECT + Company, + COUNT(*) as employee_count, + AVG(CAST(SUBSTR(Connected, 1, INSTR(Connected, ' ') - 1) AS INTEGER)) as avg_connection_days + FROM df + WHERE Position LIKE '%Engineer%' + GROUP BY Company + HAVING employee_count > 5 + ORDER BY employee_count DESC, avg_connection_days DESC + LIMIT 10 + """ + + start_time = time.time() + result = data_retriever.forward(query=query) + end_time = time.time() + + assert len(result) == 10, "Query should return top 10 companies" + assert end_time - start_time < 10, "Complex query should execute in less than 10 seconds" + assert all('Company' in row and 'employee_count' in row and 'avg_connection_days' in row for row in result) + +def test_multiple_queries_performance(data_retriever): + queries = [ + "SELECT COUNT(*) as total FROM df", + "SELECT Position, COUNT(*) as count FROM df GROUP BY Position ORDER BY count DESC LIMIT 5", + "SELECT Company, AVG(CAST(SUBSTR(Connected, 1, INSTR(Connected, ' ') - 1) AS INTEGER)) as avg_days FROM df GROUP BY Company HAVING avg_days > 365 ORDER BY avg_days DESC LIMIT 10", + "SELECT * FROM df WHERE Position LIKE '%Data Scientist%' AND Company IN (SELECT Company FROM df GROUP BY Company HAVING COUNT(*) > 10)", + ] + + start_time = time.time() + results = [data_retriever.forward(query=query) for query in queries] + end_time = time.time() + + assert len(results) == 4, "All queries should execute successfully" + assert end_time - start_time < 20, "Multiple complex queries should execute in less than 20 seconds" + +def test_large_result_set(data_retriever): + query = "SELECT * FROM df WHERE Connected LIKE '%year%'" + + start_time = time.time() + result = data_retriever.forward(query=query) + end_time = time.time() + + assert len(result) > 1000, "Large result set should contain over 1000 records" + assert end_time - start_time < 15, "Large result set query should execute in less than 15 seconds" + +def test_memory_usage(data_retriever): + import psutil + import os + + process = psutil.Process(os.getpid()) + start_memory = process.memory_info().rss / 1024 / 1024 # Memory in MB + + # Perform a memory-intensive operation + result = data_retriever.forward() + del result # Clear the result to see memory usage after garbage collection + + end_memory = process.memory_info().rss / 1024 / 1024 # Memory in MB + memory_increase = end_memory - start_memory + + assert memory_increase < 1000, f"Memory usage increase should be less than 1000 MB, but was {memory_increase:.2f} MB" + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) \ No newline at end of file diff --git a/tests/test_event_kit_service.py b/tests/test_event_kit_service.py index 5c86d70..0d4cb32 100644 --- a/tests/test_event_kit_service.py +++ b/tests/test_event_kit_service.py @@ -1,140 +1,140 @@ -import pytest -from unittest.mock import Mock, patch -from uuid import UUID -from datetime import datetime, timedelta -from src.dspygen.experiments.event_kit.event_kit_service import EventKitService, Calendar, Reminder - -@pytest.fixture -def mock_event_kit(): - with patch('src.dspygen.experiments.event_kit.event_kit_service.EventKit') as mock_ek: - yield mock_ek - -@pytest.fixture -def event_kit_service(mock_event_kit): - return EventKitService() - -@pytest.fixture -def sample_calendar(): - return Calendar(id=UUID('12345678-1234-5678-1234-567812345678'), title='Test Calendar') - -@pytest.fixture -def sample_reminder(): - return Reminder( - id=UUID('87654321-4321-8765-4321-876543210987'), - title='Test Reminder', - due_date=datetime.now() + timedelta(days=1), - completed=False, - notes='Test notes', - calendar_id=UUID('12345678-1234-5678-1234-567812345678') - ) - -def test_request_access(event_kit_service, mock_event_kit): - # Test successful access request - mock_event_kit.EKEntityTypeReminder = 'reminder' - event_kit_service.store.requestAccessToEntityType.side_effect = lambda entity_type, completionHandler: completionHandler(True, None) - event_kit_service.request_access() - - # Test access denied - event_kit_service.store.requestAccessToEntityType.side_effect = lambda entity_type, completionHandler: completionHandler(False, None) - with pytest.raises(PermissionError): - event_kit_service.request_access() - -def test_get_calendars(event_kit_service, mock_event_kit, sample_calendar): - mock_calendar = Mock() - mock_calendar.calendarIdentifier.return_value = str(sample_calendar.ci_id) - mock_calendar.title.return_value = sample_calendar.title - event_kit_service.store.calendarsForEntityType_.return_value = [mock_calendar] - - calendars = event_kit_service.get_calendars() - assert len(calendars) == 1 - assert calendars[0].ci_id == sample_calendar.ci_id - assert calendars[0].title == sample_calendar.title - -def test_get_reminders(event_kit_service, mock_event_kit, sample_calendar, sample_reminder): - mock_calendar = Mock() - mock_calendar.calendarIdentifier.return_value = str(sample_calendar.ci_id) - event_kit_service.store.calendarsForEntityType_.return_value = [mock_calendar] - - mock_reminder = Mock() - mock_reminder.calendarItemIdentifier.return_value = str(sample_reminder.ci_id) - mock_reminder.title.return_value = sample_reminder.title - mock_reminder.dueDateComponents.return_value.date.return_value = sample_reminder.due_date - mock_reminder.isCompleted.return_value = sample_reminder.completed - mock_reminder.notes.return_value = sample_reminder.notes - - event_kit_service.store.remindersMatchingPredicate_.return_value = [mock_reminder] - - reminders = event_kit_service.get_reminders(sample_calendar.ci_id) - assert len(reminders) == 1 - assert reminders[0].ci_id == sample_reminder.ci_id - assert reminders[0].title == sample_reminder.title - assert reminders[0].due_date == sample_reminder.due_date - assert reminders[0].completed == sample_reminder.completed - assert reminders[0].notes == sample_reminder.notes - assert reminders[0].calendar_id == sample_calendar.ci_id - -def test_add_reminder(event_kit_service, mock_event_kit, sample_calendar, sample_reminder): - mock_calendar = Mock() - mock_calendar.calendarIdentifier.return_value = str(sample_calendar.ci_id) - event_kit_service.store.calendarsForEntityType_.return_value = [mock_calendar] - - mock_new_reminder = Mock() - mock_new_reminder.calendarItemIdentifier.return_value = str(sample_reminder.ci_id) - mock_new_reminder.title.return_value = sample_reminder.title - mock_new_reminder.dueDateComponents.return_value.date.return_value = sample_reminder.due_date - mock_new_reminder.isCompleted.return_value = sample_reminder.completed - mock_new_reminder.notes.return_value = sample_reminder.notes - - mock_event_kit.EKReminder.reminderWithEventStore_.return_value = mock_new_reminder - - added_reminder = event_kit_service.add_reminder(sample_calendar.ci_id, sample_reminder) - assert added_reminder.ci_id == sample_reminder.ci_id - assert added_reminder.title == sample_reminder.title - assert added_reminder.due_date == sample_reminder.due_date - assert added_reminder.completed == sample_reminder.completed - assert added_reminder.notes == sample_reminder.notes - assert added_reminder.calendar_id == sample_calendar.ci_id - - event_kit_service.store.saveReminder_commit_error_.assert_called_once() - -def test_update_reminder(event_kit_service, mock_event_kit, sample_calendar, sample_reminder): - event_kit_service.get_reminders = Mock(return_value=[sample_reminder]) - event_kit_service.add_reminder = Mock(return_value=sample_reminder) - - updated_reminder_data = sample_reminder.copy() - updated_reminder_data.title = "Updated Title" - updated_reminder_data.completed = True - - updated_reminder = event_kit_service.update_reminder(sample_calendar.ci_id, sample_reminder.ci_id, updated_reminder_data) - - assert updated_reminder.ci_id == sample_reminder.ci_id - assert updated_reminder.title == "Updated Title" - assert updated_reminder.completed == True - event_kit_service.add_reminder.assert_called_once() - -def test_delete_reminder(event_kit_service, mock_event_kit, sample_calendar, sample_reminder): - event_kit_service.get_reminders = Mock(return_value=[sample_reminder]) - mock_reminder_to_delete = Mock() - event_kit_service.store.calendarItemWithIdentifier_.return_value = mock_reminder_to_delete - - event_kit_service.delete_reminder(sample_calendar.ci_id, sample_reminder.ci_id) - - event_kit_service.store.removeReminder_commit_error_.assert_called_once_with(mock_reminder_to_delete, True, None) - -def test_reminder_not_found(event_kit_service, sample_calendar): - event_kit_service.get_reminders = Mock(return_value=[]) - - with pytest.raises(StopIteration): - event_kit_service.update_reminder(sample_calendar.ci_id, UUID('00000000-0000-0000-0000-000000000000'), Reminder(title="Non-existent Reminder", calendar_id=sample_calendar.ci_id)) - - with pytest.raises(StopIteration): - event_kit_service.delete_reminder(sample_calendar.ci_id, UUID('00000000-0000-0000-0000-000000000000')) - -def test_calendar_not_found(event_kit_service, mock_event_kit): - event_kit_service.store.calendarsForEntityType_.return_value = [] - - with pytest.raises(StopIteration): - event_kit_service.get_reminders(UUID('00000000-0000-0000-0000-000000000000')) - - with pytest.raises(StopIteration): - event_kit_service.add_reminder(UUID('00000000-0000-0000-0000-000000000000'), Reminder(title="Test Reminder", calendar_id=UUID('00000000-0000-0000-0000-000000000000'))) \ No newline at end of file +# import pytest +# from unittest.mock import Mock, patch +# from uuid import UUID +# from datetime import datetime, timedelta +# from src.dspygen.experiments.event_kit.event_kit_service import EventKitService, Calendar, Reminder +# +# @pytest.fixture +# def mock_event_kit(): +# with patch('src.dspygen.experiments.event_kit.event_kit_service.EventKit') as mock_ek: +# yield mock_ek +# +# @pytest.fixture +# def event_kit_service(mock_event_kit): +# return EventKitService() +# +# @pytest.fixture +# def sample_calendar(): +# return Calendar(id=UUID('12345678-1234-5678-1234-567812345678'), title='Test Calendar') +# +# @pytest.fixture +# def sample_reminder(): +# return Reminder( +# id=UUID('87654321-4321-8765-4321-876543210987'), +# title='Test Reminder', +# due_date=datetime.now() + timedelta(days=1), +# completed=False, +# notes='Test notes', +# calendar_id=UUID('12345678-1234-5678-1234-567812345678') +# ) +# +# def test_request_access(event_kit_service, mock_event_kit): +# # Test successful access request +# mock_event_kit.EKEntityTypeReminder = 'reminder' +# event_kit_service.store.requestAccessToEntityType.side_effect = lambda entity_type, completionHandler: completionHandler(True, None) +# event_kit_service.request_access() +# +# # Test access denied +# event_kit_service.store.requestAccessToEntityType.side_effect = lambda entity_type, completionHandler: completionHandler(False, None) +# with pytest.raises(PermissionError): +# event_kit_service.request_access() +# +# def test_get_calendars(event_kit_service, mock_event_kit, sample_calendar): +# mock_calendar = Mock() +# mock_calendar.calendarIdentifier.return_value = str(sample_calendar.ci_id) +# mock_calendar.title.return_value = sample_calendar.title +# event_kit_service.store.calendarsForEntityType_.return_value = [mock_calendar] +# +# calendars = event_kit_service.get_calendars() +# assert len(calendars) == 1 +# assert calendars[0].ci_id == sample_calendar.ci_id +# assert calendars[0].title == sample_calendar.title +# +# def test_get_reminders(event_kit_service, mock_event_kit, sample_calendar, sample_reminder): +# mock_calendar = Mock() +# mock_calendar.calendarIdentifier.return_value = str(sample_calendar.ci_id) +# event_kit_service.store.calendarsForEntityType_.return_value = [mock_calendar] +# +# mock_reminder = Mock() +# mock_reminder.calendarItemIdentifier.return_value = str(sample_reminder.ci_id) +# mock_reminder.title.return_value = sample_reminder.title +# mock_reminder.dueDateComponents.return_value.date.return_value = sample_reminder.due_date +# mock_reminder.isCompleted.return_value = sample_reminder.completed +# mock_reminder.notes.return_value = sample_reminder.notes +# +# event_kit_service.store.remindersMatchingPredicate_.return_value = [mock_reminder] +# +# reminders = event_kit_service.get_reminders(sample_calendar.ci_id) +# assert len(reminders) == 1 +# assert reminders[0].ci_id == sample_reminder.ci_id +# assert reminders[0].title == sample_reminder.title +# assert reminders[0].due_date == sample_reminder.due_date +# assert reminders[0].completed == sample_reminder.completed +# assert reminders[0].notes == sample_reminder.notes +# assert reminders[0].calendar_id == sample_calendar.ci_id +# +# def test_add_reminder(event_kit_service, mock_event_kit, sample_calendar, sample_reminder): +# mock_calendar = Mock() +# mock_calendar.calendarIdentifier.return_value = str(sample_calendar.ci_id) +# event_kit_service.store.calendarsForEntityType_.return_value = [mock_calendar] +# +# mock_new_reminder = Mock() +# mock_new_reminder.calendarItemIdentifier.return_value = str(sample_reminder.ci_id) +# mock_new_reminder.title.return_value = sample_reminder.title +# mock_new_reminder.dueDateComponents.return_value.date.return_value = sample_reminder.due_date +# mock_new_reminder.isCompleted.return_value = sample_reminder.completed +# mock_new_reminder.notes.return_value = sample_reminder.notes +# +# mock_event_kit.EKReminder.reminderWithEventStore_.return_value = mock_new_reminder +# +# added_reminder = event_kit_service.add_reminder(sample_calendar.ci_id, sample_reminder) +# assert added_reminder.ci_id == sample_reminder.ci_id +# assert added_reminder.title == sample_reminder.title +# assert added_reminder.due_date == sample_reminder.due_date +# assert added_reminder.completed == sample_reminder.completed +# assert added_reminder.notes == sample_reminder.notes +# assert added_reminder.calendar_id == sample_calendar.ci_id +# +# event_kit_service.store.saveReminder_commit_error_.assert_called_once() +# +# def test_update_reminder(event_kit_service, mock_event_kit, sample_calendar, sample_reminder): +# event_kit_service.get_reminders = Mock(return_value=[sample_reminder]) +# event_kit_service.add_reminder = Mock(return_value=sample_reminder) +# +# updated_reminder_data = sample_reminder.copy() +# updated_reminder_data.title = "Updated Title" +# updated_reminder_data.completed = True +# +# updated_reminder = event_kit_service.update_reminder(sample_calendar.ci_id, sample_reminder.ci_id, updated_reminder_data) +# +# assert updated_reminder.ci_id == sample_reminder.ci_id +# assert updated_reminder.title == "Updated Title" +# assert updated_reminder.completed == True +# event_kit_service.add_reminder.assert_called_once() +# +# def test_delete_reminder(event_kit_service, mock_event_kit, sample_calendar, sample_reminder): +# event_kit_service.get_reminders = Mock(return_value=[sample_reminder]) +# mock_reminder_to_delete = Mock() +# event_kit_service.store.calendarItemWithIdentifier_.return_value = mock_reminder_to_delete +# +# event_kit_service.delete_reminder(sample_calendar.ci_id, sample_reminder.ci_id) +# +# event_kit_service.store.removeReminder_commit_error_.assert_called_once_with(mock_reminder_to_delete, True, None) +# +# def test_reminder_not_found(event_kit_service, sample_calendar): +# event_kit_service.get_reminders = Mock(return_value=[]) +# +# with pytest.raises(StopIteration): +# event_kit_service.update_reminder(sample_calendar.ci_id, UUID('00000000-0000-0000-0000-000000000000'), Reminder(title="Non-existent Reminder", calendar_id=sample_calendar.ci_id)) +# +# with pytest.raises(StopIteration): +# event_kit_service.delete_reminder(sample_calendar.ci_id, UUID('00000000-0000-0000-0000-000000000000')) +# +# def test_calendar_not_found(event_kit_service, mock_event_kit): +# event_kit_service.store.calendarsForEntityType_.return_value = [] +# +# with pytest.raises(StopIteration): +# event_kit_service.get_reminders(UUID('00000000-0000-0000-0000-000000000000')) +# +# with pytest.raises(StopIteration): +# event_kit_service.add_reminder(UUID('00000000-0000-0000-0000-000000000000'), Reminder(title="Test Reminder", calendar_id=UUID('00000000-0000-0000-0000-000000000000'))) \ No newline at end of file diff --git a/tests/test_integration_email_responder_workflow.py b/tests/test_integration_email_responder_workflow.py new file mode 100644 index 0000000..5fceeaf --- /dev/null +++ b/tests/test_integration_email_responder_workflow.py @@ -0,0 +1,122 @@ +import pytest +import os +from unittest.mock import patch, MagicMock +from dspygen.subcommands.wkf_cmd import run_workflows_in_directory, app +from dspygen.pyautomator.linkedin.linkedin_app import LinkedInApp +from dspygen.modules.automated_email_responder_module import AutomatedEmailResponderModule +from typer.testing import CliRunner + +@pytest.fixture +def mock_linkedin_app(): + return MagicMock(spec=LinkedInApp) + +@pytest.fixture +def mock_email_responder(): + return MagicMock(spec=AutomatedEmailResponderModule) + +@pytest.fixture +def sample_workflow_yaml(tmp_path): + workflow_content = """ + name: email_responder_workflow + schedule: "*/30 * * * *" + tasks: + - name: fetch_linkedin_profile + module: LinkedInApp + method: get_profile_markdown + args: + profile_url: "https://www.linkedin.com/in/example" + output_file: "profile.md" + - name: respond_to_email + module: AutomatedEmailResponderModule + method: forward + args: + email_message: "Hello, I'd like to discuss a job opportunity." + linkedin_profile: "{{tasks.fetch_linkedin_profile.output}}" + """ + workflow_file = tmp_path / "email_responder_workflow.yaml" + workflow_file.write_text(workflow_content) + return str(workflow_file) + +def test_integration_workflow_execution(sample_workflow_yaml, mock_linkedin_app, mock_email_responder): + with patch('dspygen.subcommands.wkf_cmd.Workflow.from_yaml') as mock_from_yaml, \ + patch('dspygen.subcommands.wkf_cmd.execute_workflow') as mock_execute, \ + patch('dspygen.experiments.pyautomator.linkedin_app.LinkedInApp', return_value=mock_linkedin_app), \ + patch('dspygen.modules.automated_email_responder_module.AutomatedEmailResponderModule', return_value=mock_email_responder): + + # Set up mock returns + mock_linkedin_app.get_profile_markdown.return_value = "Mocked LinkedIn Profile" + mock_email_responder.forward.return_value = "Mocked Email Response" + + # Run the workflow + scheduler = run_workflows_in_directory(os.path.dirname(sample_workflow_yaml)) + + # Assert that the scheduler was created and has one job + assert scheduler is not None + assert len(scheduler.get_jobs()) == 1 + + # Trigger the job execution + job = scheduler.get_jobs()[0] + job.func() + + # Verify that the LinkedIn profile was fetched + mock_linkedin_app.get_profile_markdown.assert_called_once_with( + profile_url="https://www.linkedin.com/in/example", + output_file="profile.md" + ) + + # Verify that the email responder was called with the correct arguments + mock_email_responder.forward.assert_called_once_with( + email_message="Hello, I'd like to discuss a job opportunity.", + linkedin_profile="Mocked LinkedIn Profile" + ) + +def test_integration_cli_trigger(sample_workflow_yaml, mock_linkedin_app, mock_email_responder): + runner = CliRunner() + + with patch('dspygen.subcommands.wkf_cmd.Workflow.from_yaml') as mock_from_yaml, \ + patch('dspygen.subcommands.wkf_cmd.execute_workflow') as mock_execute, \ + patch('dspygen.experiments.pyautomator.linkedin_app.LinkedInApp', return_value=mock_linkedin_app), \ + patch('dspygen.modules.automated_email_responder_module.AutomatedEmailResponderModule', return_value=mock_email_responder): + + # Set up mock returns + mock_linkedin_app.get_profile_markdown.return_value = "Mocked LinkedIn Profile" + mock_email_responder.forward.return_value = "Mocked Email Response" + + # Trigger the workflow using the CLI + result = runner.invoke(app, ["trigger", os.path.basename(sample_workflow_yaml)[:-5]]) + + assert result.exit_code == 0 + assert "Workflow email_responder_workflow triggered" in result.output + + # Verify that the LinkedIn profile was fetched + mock_linkedin_app.get_profile_markdown.assert_called_once() + + # Verify that the email responder was called + mock_email_responder.forward.assert_called_once() + +def test_integration_error_handling(sample_workflow_yaml, mock_linkedin_app, mock_email_responder): + with patch('dspygen.subcommands.wkf_cmd.Workflow.from_yaml') as mock_from_yaml, \ + patch('dspygen.subcommands.wkf_cmd.execute_workflow') as mock_execute, \ + patch('dspygen.experiments.pyautomator.linkedin_app.LinkedInApp', return_value=mock_linkedin_app), \ + patch('dspygen.modules.automated_email_responder_module.AutomatedEmailResponderModule', return_value=mock_email_responder): + + # Simulate an error in LinkedIn profile fetching + mock_linkedin_app.get_profile_markdown.side_effect = Exception("Network error") + + # Run the workflow + scheduler = run_workflows_in_directory(os.path.dirname(sample_workflow_yaml)) + + # Trigger the job execution + job = scheduler.get_jobs()[0] + job.func() + + # Verify that the LinkedIn profile fetch was attempted + mock_linkedin_app.get_profile_markdown.assert_called_once() + + # Verify that the email responder was not called due to the error + mock_email_responder.forward.assert_not_called() + + # You might want to add assertions here to check if the error was logged or handled appropriately + +if __name__ == "__main__": + pytest.main() \ No newline at end of file diff --git a/tests/test_ultra_advanced_integration_job_search_workflow.py b/tests/test_ultra_advanced_integration_job_search_workflow.py new file mode 100644 index 0000000..8c95d76 --- /dev/null +++ b/tests/test_ultra_advanced_integration_job_search_workflow.py @@ -0,0 +1,181 @@ +# import pytest +# import os +# from unittest.mock import patch, MagicMock +# from dspygen.subcommands.wkf_cmd import run_workflows_in_directory, app +# from dspygen.experiments.pyautomator.linkedin_app import LinkedInApp +# from dspygen.modules.automated_email_responder_module import AutomatedEmailResponderModule, AutomatedEmailResponderSignature +# from dspygen.ai.assistant import AIAssistant +# from dspygen.task_management import TaskManager +# from dspygen.utils.dspy_tools import init_dspy +# from dspygen.subcommands.wrt_cmd import new_rm +# from dspygen.modules.ask_data_module import AskDataModule +# from typer.testing import CliRunner +# from apscheduler.schedulers.background import BackgroundScheduler +# import dspy +# +# @pytest.fixture +# def mock_linkedin_app(): +# return MagicMock(spec=LinkedInApp) +# +# @pytest.fixture +# def mock_email_responder(): +# return MagicMock(spec=AutomatedEmailResponderModule) +# +# @pytest.fixture +# def mock_ai_assistant(): +# return MagicMock(spec=AIAssistant) +# +# @pytest.fixture +# def mock_task_manager(): +# return MagicMock(spec=TaskManager) +# +# @pytest.fixture +# def mock_ask_data_module(): +# return MagicMock(spec=AskDataModule) +# +# @pytest.fixture +# def sample_ultra_complex_workflow_yaml(tmp_path): +# workflow_content = """ +# name: ultra_advanced_job_search_workflow +# schedule: "*/10 * * * *" +# tasks: +# - name: analyze_job_market +# module: AIAssistant +# method: analyze_job_market_trends +# args: +# industries: ["Tech", "Finance", "Healthcare"] +# - name: fetch_linkedin_profile +# module: LinkedInApp +# method: get_profile_markdown +# args: +# profile_url: "https://www.linkedin.com/in/example" +# - name: generate_personalized_messages +# module: AIAssistant +# method: generate_message_chain +# args: +# profile: "{{tasks.fetch_linkedin_profile.output}}" +# market_trends: "{{tasks.analyze_job_market.output}}" +# - name: send_personalized_messages +# module: AutomatedEmailResponderModule +# method: send_bulk_messages +# args: +# messages: "{{tasks.generate_personalized_messages.output}}" +# - name: analyze_response_sentiment +# module: AIAssistant +# method: analyze_response_sentiment +# args: +# responses: "{{tasks.send_personalized_messages.output.responses}}" +# - name: update_task_list +# module: TaskManager +# method: update_tasks +# args: +# new_tasks: "{{tasks.analyze_response_sentiment.output.suggested_actions}}" +# - name: optimize_strategy +# module: AIAssistant +# method: optimize_job_search_strategy +# args: +# market_trends: "{{tasks.analyze_job_market.output}}" +# sentiment_analysis: "{{tasks.analyze_response_sentiment.output}}" +# current_tasks: "{{tasks.update_task_list.output}}" +# - name: analyze_job_postings +# module: AskDataModule +# method: forward +# args: +# question: "What are the top 5 most common skills required in recent job postings?" +# file_path: "recent_job_postings.csv" +# """ +# workflow_file = tmp_path / "ultra_advanced_job_search_workflow.yaml" +# workflow_file.write_text(workflow_content) +# return str(workflow_file) +# +# def test_ultra_advanced_integration_workflow_execution(sample_ultra_complex_workflow_yaml, mock_linkedin_app, mock_email_responder, mock_ai_assistant, mock_task_manager, mock_ask_data_module): +# with patch('dspygen.subcommands.wkf_cmd.Workflow.from_yaml') as mock_from_yaml, \ +# patch('dspygen.subcommands.wkf_cmd.execute_workflow') as mock_execute, \ +# patch('dspygen.experiments.pyautomator.linkedin_app.LinkedInApp', return_value=mock_linkedin_app), \ +# patch('dspygen.modules.automated_email_responder_module.AutomatedEmailResponderModule', return_value=mock_email_responder), \ +# patch('dspygen.ai.assistant.AIAssistant', return_value=mock_ai_assistant), \ +# patch('dspygen.task_management.TaskManager', return_value=mock_task_manager), \ +# patch('dspygen.modules.ask_data_module.AskDataModule', return_value=mock_ask_data_module): +# +# # Set up mock returns +# mock_ai_assistant.analyze_job_market_trends.return_value = { +# "top_industries": ["Tech", "Finance"], +# "emerging_skills": ["AI", "Blockchain"], +# "market_outlook": "Positive growth in tech sector" +# } +# mock_linkedin_app.get_profile_markdown.return_value = "John Doe | Software Engineer | Skills: Python, AI, Cloud Computing" +# mock_ai_assistant.generate_message_chain.return_value = [ +# {"recipient": "John Doe", "message": "Personalized message for John"}, +# {"recipient": "Jane Smith", "message": "Personalized message for Jane"} +# ] +# mock_email_responder.send_bulk_messages.return_value = { +# "sent": 2, +# "failed": 0, +# "responses": ["Interested, let's talk", "Thanks, but not now"] +# } +# mock_ai_assistant.analyze_response_sentiment.return_value = { +# "overall_sentiment": "Positive", +# "suggested_actions": ["Schedule call with John Doe", "Follow up with Jane Smith in 3 months"] +# } +# mock_task_manager.update_tasks.return_value = ["Schedule call with John Doe", "Follow up with Jane Smith in 3 months"] +# mock_ai_assistant.optimize_job_search_strategy.return_value = { +# "focus_industries": ["Tech"], +# "skill_development": ["AI"], +# "networking_strategy": "Increase outreach to CTOs in tech industry" +# } +# mock_ask_data_module.forward.return_value = "Top 5 skills: 1. Python, 2. Machine Learning, 3. SQL, 4. Cloud Computing, 5. Data Analysis" +# +# # Run the workflow +# scheduler = run_workflows_in_directory(os.path.dirname(sample_ultra_complex_workflow_yaml)) +# +# assert scheduler is not None +# assert len(scheduler.get_jobs()) == 1 +# +# # Trigger the job execution +# job = scheduler.get_jobs()[0] +# job.func() +# +# # Verify each step of the workflow +# mock_ai_assistant.analyze_job_market_trends.assert_called_once() +# mock_linkedin_app.get_profile_markdown.assert_called_once() +# mock_ai_assistant.generate_message_chain.assert_called_once() +# mock_email_responder.send_bulk_messages.assert_called_once() +# mock_ai_assistant.analyze_response_sentiment.assert_called_once() +# mock_task_manager.update_tasks.assert_called_once() +# mock_ai_assistant.optimize_job_search_strategy.assert_called_once() +# mock_ask_data_module.forward.assert_called_once_with( +# question="What are the top 5 most common skills required in recent job postings?", +# file_path="recent_job_postings.csv" +# ) +# +# def test_writer_module_integration(): +# runner = CliRunner() +# with runner.isolated_filesystem(): +# result = runner.invoke(new_rm, ["JobSearchWriter"]) +# assert result.exit_code == 0 +# assert "job_search_writer.py" in result.output +# +# def test_linkedin_profile_fetching(mock_linkedin_app): +# profile_url = "https://www.linkedin.com/in/example" +# mock_linkedin_app.get_profile_markdown.return_value = "John Doe | Software Engineer | Skills: Python, AI, Cloud Computing" +# +# profile_content = mock_linkedin_app.get_profile_markdown(profile_url) +# +# assert "John Doe" in profile_content +# assert "Software Engineer" in profile_content +# assert "Python" in profile_content +# +# def test_ask_data_module_integration(mock_ask_data_module): +# question = "What are the most in-demand skills for data scientists?" +# file_path = "job_market_data.csv" +# expected_answer = "The most in-demand skills for data scientists are: 1. Python, 2. Machine Learning, 3. SQL, 4. Data Visualization, 5. Big Data technologies" +# +# mock_ask_data_module.forward.return_value = expected_answer +# +# answer = mock_ask_data_module.forward(question=question, file_path=file_path) +# +# assert answer == expected_answer +# mock_ask_data_module.forward.assert_called_once_with(question=question, file_path=file_path) +# +# if __name__ == "__main__": +# pytest.main([__file__, "-v"]) \ No newline at end of file diff --git a/tests/test_wkf_cmd.py b/tests/test_wkf_cmd.py new file mode 100644 index 0000000..499c893 --- /dev/null +++ b/tests/test_wkf_cmd.py @@ -0,0 +1,86 @@ +import pytest +from typer.testing import CliRunner +from unittest.mock import patch, MagicMock +from dspygen.subcommands.wkf_cmd import app, run_workflows_in_directory +from dspygen.workflow.workflow_models import Workflow +from apscheduler.schedulers.background import BackgroundScheduler + +runner = CliRunner() + +@pytest.fixture +def mock_workflow(): + return MagicMock(spec=Workflow) + +@pytest.fixture +def mock_scheduler(): + return MagicMock(spec=BackgroundScheduler) + +def test_run_workflow(mock_workflow): + with patch('dspygen.subcommands.wkf_cmd.Workflow.from_yaml', return_value=mock_workflow) as mock_from_yaml, \ + patch('dspygen.subcommands.wkf_cmd.execute_workflow') as mock_execute: + + result = runner.invoke(app, ["run", "test_workflow.yaml"]) + + assert result.exit_code == 0 + mock_from_yaml.assert_called_once_with("test_workflow.yaml") + mock_execute.assert_called_once_with(mock_workflow) + +def test_run_workflows_in_directory(tmp_path, mock_workflow, mock_scheduler): + # Create test YAML files + (tmp_path / "workflow1.yaml").write_text("# Test workflow 1") + (tmp_path / "subdir").mkdir() + (tmp_path / "subdir" / "workflow2.yaml").write_text("# Test workflow 2") + + with patch('dspygen.subcommands.wkf_cmd.Workflow.from_yaml', return_value=mock_workflow), \ + patch('dspygen.subcommands.wkf_cmd.BackgroundScheduler', return_value=mock_scheduler), \ + patch('dspygen.subcommands.wkf_cmd.schedule_workflow') as mock_schedule: + + # Test with recursion + scheduler = run_workflows_in_directory(str(tmp_path), recursive=True) + assert scheduler == mock_scheduler + assert mock_schedule.call_count == 2 + + # Reset mocks + mock_schedule.reset_mock() + mock_scheduler.reset_mock() + + # Test without recursion + scheduler = run_workflows_in_directory(str(tmp_path), recursive=False) + assert scheduler == mock_scheduler + assert mock_schedule.call_count == 1 + + # Test with empty directory + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + with patch('dspygen.subcommands.wkf_cmd.BackgroundScheduler', return_value=mock_scheduler): + scheduler = run_workflows_in_directory(str(empty_dir)) + assert scheduler is None + mock_scheduler.shutdown.assert_called_once() + +@pytest.mark.parametrize("recursive", [True, False]) +def test_run_all_command(recursive, mock_scheduler): + with patch('dspygen.subcommands.wkf_cmd.run_workflows_in_directory', return_value=mock_scheduler) as mock_run: + result = runner.invoke(app, ["run-all", ".", f"--{'no-' if not recursive else ''}recursive"]) + + assert result.exit_code == 0 + mock_run.assert_called_once_with(".", recursive=recursive) + mock_scheduler.start.assert_called_once() + mock_scheduler.print_jobs.assert_called_once() + +def test_run_all_command_no_workflows(): + with patch('dspygen.subcommands.wkf_cmd.run_workflows_in_directory', return_value=None) as mock_run: + result = runner.invoke(app, ["run-all", "."]) + + assert result.exit_code == 0 + +def test_trigger_workflow(mock_workflow): + with patch('dspygen.subcommands.wkf_cmd.Workflow.from_yaml', return_value=mock_workflow) as mock_from_yaml, \ + patch('dspygen.subcommands.wkf_cmd.execute_workflow') as mock_execute, \ + patch('dspygen.subcommands.wkf_cmd.os.path.exists', return_value=True): + + result = runner.invoke(app, ["trigger", "test_workflow"]) + + assert result.exit_code == 0 + mock_from_yaml.assert_called_once() + mock_execute.assert_called_once_with(mock_workflow) + assert "Workflow test_workflow triggered" in result.output diff --git a/tests/test_workflow_integration.py b/tests/test_workflow_integration.py new file mode 100644 index 0000000..bb7c311 --- /dev/null +++ b/tests/test_workflow_integration.py @@ -0,0 +1,60 @@ +import os +import pytest +from dspygen.workflow.workflow_models import Workflow, DateTrigger +from dspygen.workflow.workflow_executor import execute_workflow, schedule_workflow +from apscheduler.schedulers.background import BackgroundScheduler +from datetime import datetime, timedelta +import time +from loguru import logger + +# Add this at the beginning of the file to set up verbose logging +logger.add("test_workflow_integration.log", level="DEBUG", rotation="1 MB") + +@pytest.fixture +def workflow_yaml(tmp_path): + output_file = tmp_path / "test_output.txt" + logger.debug(f"Creating workflow YAML with output file: {output_file}") + return f""" +name: TestWorkflow +triggers: + - type: date + run_date: now +jobs: + - name: TestJob + runner: python + steps: + - name: TestAction + code: | + print("hello world") + with open('{output_file}', 'w') as f: + f.write('Integration test successful') + """ + +@pytest.fixture(scope="module") +def scheduler(): + logger.debug("Creating BackgroundScheduler") + scheduler = BackgroundScheduler() + scheduler.start() + yield scheduler + logger.debug("Shutting down BackgroundScheduler") + scheduler.shutdown() + +def test_workflow_execution_from_yaml(workflow_yaml, tmp_path): + logger.info("Starting test_workflow_execution_from_yaml") + yaml_path = tmp_path / "test_workflow.yaml" + with open(yaml_path, "w") as f: + f.write(workflow_yaml) + logger.debug(f"Saved workflow YAML to: {yaml_path}") + + workflow = Workflow.from_yaml(str(yaml_path)) + logger.debug(f"Created workflow object: {workflow}") + execute_workflow(workflow) + + output_file = tmp_path / "test_output.txt" + logger.debug(f"Checking output file: {output_file}") + with open(output_file, "r") as f: + content = f.read() + logger.info(f"Output file content: {content}") + assert content == "Integration test successful" + logger.info("test_workflow_execution_from_yaml completed successfully") + diff --git a/tests/test_workflow_runner.py b/tests/test_workflow_runner.py new file mode 100644 index 0000000..685fa92 --- /dev/null +++ b/tests/test_workflow_runner.py @@ -0,0 +1,24 @@ +import pytest +from dspygen.subcommands.wkf_cmd import run_workflows_in_directory + +def test_run_workflows_in_directory(tmp_path): + # Create a temporary directory structure with some YAML files + (tmp_path / "workflow1.yaml").write_text("# Test workflow 1") + (tmp_path / "subdir").mkdir() + (tmp_path / "subdir" / "workflow2.yaml").write_text("# Test workflow 2") + + # Test with recursion + scheduler = run_workflows_in_directory(str(tmp_path), recursive=True) + assert scheduler is not None + assert len(scheduler.get_jobs()) == 2 + + # Test without recursion + scheduler = run_workflows_in_directory(str(tmp_path), recursive=False) + assert scheduler is not None + assert len(scheduler.get_jobs()) == 1 + + # Test with empty directory + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + scheduler = run_workflows_in_directory(str(empty_dir)) + assert scheduler is None \ No newline at end of file diff --git a/tests/test_workflow_scheduler.py b/tests/test_workflow_scheduler.py new file mode 100644 index 0000000..3c5f92f --- /dev/null +++ b/tests/test_workflow_scheduler.py @@ -0,0 +1,115 @@ +import pytest +from datetime import datetime, timedelta +from apscheduler.schedulers.background import BackgroundScheduler +from dspygen.workflow.workflow_models import Workflow, Job, Action, CronTrigger +from dspygen.workflow.workflow_executor import schedule_workflow +import pytz + +class MockClock: + def __init__(self, initial): + self.current = initial.replace(tzinfo=pytz.UTC) + + def get_current_time(self): + return self.current + + def advance(self, delta): + self.current += delta + +@pytest.fixture +def mock_clock(): + return MockClock(datetime(2023, 1, 1, 0, 0, 0)) + +@pytest.fixture +def scheduler(mock_clock): + scheduler = BackgroundScheduler() + scheduler.configure(clock=mock_clock.get_current_time) + return scheduler + +def test_cron_trigger_simulation(scheduler, mock_clock): + workflow = Workflow( + name="TestWorkflow", + triggers=[CronTrigger(cron="*/5 * * * *")], # Every 5 minutes + jobs=[ + Job( + name="TestJob", + runner="python", + steps=[Action(name="TestAction", code="print('Job executed')")] + ) + ] + ) + + schedule_workflow(workflow, scheduler) + scheduler.start() + + execution_times = [] + + # Simulate 1 hour passing + for _ in range(12): # 12 * 5 minutes = 1 hour + mock_clock.advance(timedelta(minutes=5)) + scheduler.wakeup() + jobs = scheduler.get_jobs() + for job in jobs: + next_run_time = job.trigger.get_next_fire_time(None, mock_clock.current) + if next_run_time and next_run_time <= mock_clock.current: + execution_times.append(mock_clock.current) + job.func(*job.args, **job.kwargs) + + scheduler.shutdown() + + # Assert that the job was executed 12 times (every 5 minutes for 1 hour) + assert len(execution_times) == 12 + + # Check that executions happened at 5-minute intervals + for i in range(1, len(execution_times)): + assert execution_times[i] - execution_times[i-1] == timedelta(minutes=5) + +def test_daily_trigger_simulation(scheduler, mock_clock): + workflow = Workflow( + name="DailyWorkflow", + triggers=[CronTrigger(cron="0 12 * * *")], # Every day at noon + jobs=[ + Job( + name="DailyJob", + runner="python", + steps=[Action(name="DailyAction", code="print('Daily job executed')")] + ) + ] + ) + + schedule_workflow(workflow, scheduler) + scheduler.start() + + execution_times = [] + + # Simulate 5 days passing + for day in range(5): + # Calculate time to next noon + current_time = mock_clock.current + next_noon = current_time.replace(hour=12, minute=0, second=0, microsecond=0) + if current_time >= next_noon: + next_noon += timedelta(days=1) + time_to_advance = next_noon - current_time + + # Advance to next noon + mock_clock.advance(time_to_advance) + scheduler.wakeup() + + jobs = scheduler.get_jobs() + for job in jobs: + next_run_time = job.trigger.get_next_fire_time(None, mock_clock.current) + if next_run_time and next_run_time <= mock_clock.current: + execution_times.append(mock_clock.current) + job.func(*job.args, **job.kwargs) + + scheduler.shutdown() + + # Assert that the job was executed 5 times (once per day for 5 days) + assert len(execution_times) == 5, f"Expected 5 executions, but got {len(execution_times)}" + + # Check that executions happened at daily intervals at noon + for i, execution_time in enumerate(execution_times): + assert execution_time.hour == 12, f"Execution {i} not at noon: {execution_time}" + assert execution_time.minute == 0, f"Execution {i} not at the start of the hour: {execution_time}" + if i > 0: + time_diff = execution_time - execution_times[i-1] + assert time_diff == timedelta(days=1), f"Incorrect interval between executions {i-1} and {i}: {time_diff}" diff --git a/tests/test_workflow_unit.py b/tests/test_workflow_unit.py new file mode 100644 index 0000000..7b08541 --- /dev/null +++ b/tests/test_workflow_unit.py @@ -0,0 +1,91 @@ +import pytest +from dspygen.workflow.workflow_models import Workflow, Job, Action, CronTrigger +from dspygen.workflow.workflow_executor import execute_workflow, execute_job, execute_action + +def test_workflow_creation(): + workflow = Workflow( + name="TestWorkflow", + triggers=[CronTrigger(cron="0 0 * * *")], + jobs=[ + Job( + name="TestJob", + runner="python", + steps=[ + Action( + name="TestAction", + code="print('Hello, World!')" + ) + ] + ) + ] + ) + assert workflow.name == "TestWorkflow" + assert len(workflow.triggers) == 1 + assert isinstance(workflow.triggers[0], CronTrigger) + assert workflow.triggers[0].cron == "0 0 * * *" + assert len(workflow.jobs) == 1 + +def test_execute_action(capsys): + action = Action( + name="TestAction", + code="print('Hello, World!')" + ) + context = {} + new_context = execute_action(action, context) + captured = capsys.readouterr() + assert captured.out.strip() == "Hello, World!" + assert new_context == {} + +def test_execute_job(capsys): + job = Job( + name="TestJob", + runner="python", + steps=[ + Action( + name="TestAction1", + code="print('Action 1')" + ), + Action( + name="TestAction2", + code="print('Action 2')" + ) + ] + ) + context = {} + new_context = execute_job(job, context) + captured = capsys.readouterr() + assert "Action 1" in captured.out + assert "Action 2" in captured.out + assert new_context == {} + +def test_execute_workflow(capsys): + workflow = Workflow( + name="TestWorkflow", + triggers=[CronTrigger(cron="0 0 * * *")], + jobs=[ + Job( + name="TestJob1", + runner="python", + steps=[ + Action( + name="TestAction1", + code="print('Job 1, Action 1')" + ) + ] + ), + Job( + name="TestJob2", + runner="python", + steps=[ + Action( + name="TestAction2", + code="print('Job 2, Action 1')" + ) + ] + ) + ] + ) + execute_workflow(workflow) + captured = capsys.readouterr() + assert "Job 1, Action 1" in captured.out + assert "Job 2, Action 1" in captured.out