complete_excel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5e5ae35f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "libraries imported\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    import psycopg2\n",
    "    import psycopg2.extras\n",
    "    import pandas as pd\n",
    "    import opendp\n",
    "    from opendp.transformations import *\n",
    "    from opendp.measurements import *\n",
    "    from opendp.mod import enable_features\n",
    "    enable_features(\"contrib\")\n",
    "    from opendp.mod import binary_search\n",
    "    from opendp.accuracy import discrete_laplacian_scale_to_accuracy\n",
    "    # Import time module, logging, os\n",
    "    import time\n",
    "    import logging\n",
    "    import os\n",
    "    print(\"libraries imported\")\n",
    "except ImportError as e:\n",
    "    print(f\"Error importing libraries: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3035ad99",
   "metadata": {},
   "outputs": [],
   "source": [
    "def postgres_DB(dbname, user, host, password):\n",
    "    try:\n",
    "        # Connect to the database\n",
    "        conn = psycopg2.connect(dbname=dbname, user=user, host=host, password=password)\n",
    "        try:\n",
    "            # Create a cursor\n",
    "            cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)\n",
    "            print(\"Connected to database, cursor defined\")\n",
    "            return conn, cur\n",
    "        except psycopg2.Error as e:\n",
    "            print(f\"Error creating cursor: {e}\")\n",
    "            # Close the connection in case of an error\n",
    "            conn.close()\n",
    "            return None, None      \n",
    "    except psycopg2.Error as e:\n",
    "        print(f\"Error connecting to database: {e}\")\n",
    "        return None, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e2e2e140",
   "metadata": {},
   "outputs": [],
   "source": [
    "def csv_scan(first_file):\n",
    "    #Scanning CSV's into dataframes\n",
    "    df = pd.read_csv(first_file)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2de5a842",
   "metadata": {},
   "outputs": [],
   "source": [
    "def postgres_scan(table_name, cursor):\n",
    "    # Execute a query to fetch column names and data from the table\n",
    "    query = f\"SELECT * FROM {table_name}\"\n",
    "    cursor.execute(query)\n",
    "\n",
    "    # Fetch column names from the cursor description\n",
    "    column_names = [desc[0] for desc in cursor.description]\n",
    "\n",
    "    # Create a DataFrame from the fetched data\n",
    "    df = pd.DataFrame(cursor.fetchall(), columns=column_names)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e9c8820b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_df(df, condition):\n",
    "    # Filter the DataFrame based on the given condition\n",
    "    filtered_df = df.query(condition)\n",
    "    # Reset index in the new dataframe\n",
    "    filtered_df.reset_index(drop=True, inplace=True)\n",
    "\n",
    "    return filtered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e6eff6c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_data(df, filtered_df, track_column):\n",
    "    # Find the maximum number of rows in the original DataFrame\n",
    "    max_rows = len(df)\n",
    "\n",
    "    # Find the difference in the number of rows\n",
    "    num_rows_diff = len(df) - len(filtered_df)\n",
    "\n",
    "    # Create a DataFrame of zeros with the same columns as low_gpa_df\n",
    "    zeros_df = pd.DataFrame(0, index=range(num_rows_diff), columns=filtered_df.columns)\n",
    "\n",
    "    # Concatenate low_gpa_df and zeros_df vertically\n",
    "    df_padded = pd.concat([filtered_df, zeros_df],ignore_index=True)\n",
    "\n",
    "    # Create a dummy column to track zero-filled rows\n",
    "    # .astype(int)converts boolean TRUE/FALSE to 1-0\n",
    "    # 0 for False(fake data), 1 for True(real data)\n",
    "    zero_filled = ~(df_padded == 0).all(axis=1)\n",
    "    df_padded[track_column] = zero_filled.astype(int)\n",
    "\n",
    "    return df_padded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "45d817b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Partially pad so that the data contains k*100 % dummy values\n",
    "def DP_pad_data(filtered_df, original_df, k, track_column):\n",
    "#     if not isinstance(filtered_df, pd.DataFrame) or not isinstance(original_df, pd.DataFrame):\n",
    "#         raise ValueError(\"Both filtered_df and original_df should be valid DataFrames.\")\n",
    "    min_size = len(filtered_df)\n",
    "    max_size = len(original_df)\n",
    "    if min_size >= max_size:\n",
    "        raise ValueError(\"filtered_df should have fewer rows than original_df.\")\n",
    "    size_diff = max_size - min_size\n",
    "    \n",
    "    if not 0 < k < 1:\n",
    "        raise ValueError(\"The value of k should be between 0 and 1.\")\n",
    "        \n",
    "    # Calculate the number of rows to pad, but ensure it does not exceed size_diff \n",
    "    # to satisfy k% dummy rows in the total size, this is the cross-formula, this could grow past max_size sometimes\n",
    "    n_rows=int((min_size * k)/(1-k))\n",
    "    \n",
    "    #when total number of rows after pad reaches max_size, meaning n_rows=size_diff #cross maths\n",
    "    max_k = size_diff / (min_size + size_diff)\n",
    "    \n",
    "    if n_rows > size_diff:\n",
    "        print(f\"Sorry, padding up to {k*100:.2f}% dummy in this dataset is not applicable, as it will exceed the size of input data.\\n\"\n",
    "              f\"Your data will be padded to the maximum input size of {max_size}.\")\n",
    "        print(f\"Maximum effective k: {max_k:.3f}, beyond this value the data will be padded to the maximum size.\\n\"\n",
    "              f\"Runtime is now equal to fully private count.\")\n",
    "        \n",
    "    #realistic pad rows might be different because we cannot exceed input (original datasize)\n",
    "    # the min funct ensures padding only til the max allowed\n",
    "    pad_rows = min(n_rows, size_diff)\n",
    "    size_df_padded = min_size + pad_rows\n",
    "    \n",
    "    zeros_df = pd.DataFrame(0, index=range(pad_rows), columns=filtered_df.columns)\n",
    "    df_padded = pd.concat([filtered_df, zeros_df], ignore_index=True)\n",
    "#     print(f\"Size padded such that the dataset has {k*100:.2f}% dummy data: {df_padded.shape[0]}\")\n",
    "#     print()\n",
    "\n",
    "    \n",
    "    zero_filled = ~(df_padded == 0).all(axis=1)\n",
    "    df_padded[track_column] = zero_filled.astype(int)\n",
    "    \n",
    "    return df_padded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fc23498a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def join(df, df2, merge_type, column_compare):\n",
    "    final_df = pd.merge(left = df, right = df2, how = merge_type, on = column_compare)\n",
    "    return final_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0303c980",
   "metadata": {},
   "outputs": [],
   "source": [
    "#count the number of real data (not padded) in a padded dataframe -> count that hides execution time\n",
    "def count_real(padded_df, track_column):\n",
    "    count_real = (padded_df[track_column] == 1).sum()    \n",
    "    #convert to int (bc how pandas handle boolean)\n",
    "    return count_real.item()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "be1910f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_real(df, column): \n",
    "    #later worry about which df\n",
    "    # Check if the column exists in the DataFrame\n",
    "    if column in df.columns:\n",
    "        # Calculate the mean for the specified column\n",
    "        mean_value = df[column].mean()\n",
    "        return mean_value\n",
    "    else:\n",
    "        raise ValueError(\"Column '{}' does not exist in the DataFrame.\".format(column_name))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "909f8f24",
   "metadata": {},
   "source": [
    "In differential privacy, sensitivity, epsilon, and scale are important concepts used in privacy-preserving mechanisms to control the amount of noise added to query results and ensure privacy protection.\n",
    "\n",
    "Sensitivity:\n",
    "Sensitivity refers to the maximum possible change in the query result when a single individual's data is added or removed from the dataset. It measures how much impact a single individual can have on the query result. The sensitivity value is specific to each query and dataset.\n",
    "\n",
    "Epsilon (ε):\n",
    "Epsilon is a privacy parameter that controls the level of privacy protection provided by a differentially private mechanism. A smaller value of epsilon provides stronger privacy guarantees but may result in noisier query results. On the other hand, a larger value of epsilon provides weaker privacy guarantees but yields less noisy query results.\n",
    "\n",
    "The range of epsilon (ε) in differential privacy is typically between 0 and infinity. However, in practice, epsilon is always a positive value, and it's common to use epsilon values that are greater than or equal to 0 and less than or equal to 1.\n",
    "\n",
    "A smaller epsilon value provides stronger privacy guarantees, meaning that the released data is more protected and less likely to reveal sensitive information about individuals in the dataset. On the other hand, a larger epsilon value provides weaker privacy guarantees, meaning that the released data may be more accurate but less private.\n",
    "\n",
    "The choice of epsilon depends on the specific privacy requirements and the level of privacy protection needed for the application. In general, smaller epsilon values are preferred for scenarios where strong privacy protection is essential, such as medical or financial data. However, larger epsilon values can be used for scenarios where a balance between privacy and data accuracy is acceptable.\n",
    "\n",
    "It's important to note that as epsilon increases, the amount of noise added to the query result decreases, which means the released data becomes less private. Therefore, the selection of an appropriate epsilon value requires a careful trade-off between privacy and utility (accuracy of the released data).\n",
    "\n",
    "Scale:\n",
    "Scale is a parameter used in the Laplace and Gaussian mechanisms to determine the amount of noise added to the query result. For the Laplace mechanism, the scale is directly proportional to sensitivity/epsilon, while for the Gaussian mechanism, the scale is sensitivity/epsilon. A higher scale value means more noise is added, which results in more privacy protection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2a704ac6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mechanism_selection():\n",
    "    print(\"Select a mechanism for adding noise:\")\n",
    "    print(\"1. Laplace Noise\")\n",
    "    print(\"2. Gaussian Noise\")\n",
    "    print(\"3. Random Noise\")\n",
    "    \n",
    "    while True:\n",
    "        choice = input(\"Enter your choice (1, 2, or 3): \")\n",
    "        if choice == \"1\":\n",
    "            return \"laplace\"\n",
    "        elif choice == \"2\":\n",
    "            return \"gaussian\"\n",
    "        elif choice == \"3\":\n",
    "            return \"random\"\n",
    "        else:\n",
    "            print(\"Invalid choice. Please enter 1, 2, or 3.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ccab7e32",
   "metadata": {},
   "outputs": [],
   "source": [
    "def privacy_parameters():\n",
    "    print(\"In order to add noise using Laplace, Gaussian or Random mechanism, first determine privacy parameters.\")\n",
    "    print(\"Enter the sensitivity (e.g., 1): \")\n",
    "    sensitivity = float(input())\n",
    "    print(\"Enter the epsilon (e.g., 0.1): \")\n",
    "    epsilon = float(input())\n",
    "    # Calculate the scale parameter for Laplace distribution\n",
    "    scale = sensitivity / epsilon\n",
    "    return sensitivity, epsilon, scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cfdd7f31",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_noise(result, mechanism, scale):\n",
    "    if mechanism == \"laplace\":\n",
    "        return laplace_noise(result,scale)\n",
    "    elif mechanism == \"gaussian\":\n",
    "        return gaussian_noise(result,scale)\n",
    "    elif mechanism == \"random\":\n",
    "        return random_noise(result,scale)\n",
    "    else:\n",
    "        raise ValueError(\"Invalid mechanism choice. Supported mechanisms are 'laplace' and 'gaussian'.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "645563bf",
   "metadata": {},
   "source": [
    "The result is a random draw from the discrete Laplace/Gaussian distribution, centered at the true count of the number of records in the underlying dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a5595c9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add Laplace noise for differential privacy\n",
    "def laplace_noise(result, scale):\n",
    "    #scale here just has to be hard-coded. Because scale is proportional but not equal to sensitivity/epsilon\n",
    "    #there isn't a universal way I've found to calculate scale for laplace without knowing standard deviation.\n",
    "    if isinstance(result, int):\n",
    "        dp_count = make_base_discrete_laplace(scale) \n",
    "    if isinstance(result, float):\n",
    "        dp_count = make_base_laplace(scale) \n",
    "    noisy_result = dp_count(result)\n",
    "    return noisy_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "41a1e7ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gaussian_noise(result,scale):\n",
    "    if isinstance(result, int):\n",
    "        dp_count = make_base_discrete_gaussian(scale)\n",
    "    if isinstance(result, float):\n",
    "        dp_count = make_base_gaussian(scale) \n",
    "    noisy_result = dp_count(result)\n",
    "    return noisy_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6a042ce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def random_noise(result, scale):\n",
    "    # Generate Laplace noise and add it to the result\n",
    "    #scale is hard-coded, explanation above\n",
    "    if isinstance(result, int):\n",
    "        noisy_result = int(result + np.random.laplace(loc=0, scale=scale))\n",
    "    if isinstance(result, float):\n",
    "        noisy_result = float(result + np.random.laplace(loc=0, scale=scale))\n",
    "    return noisy_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8be08bc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def max_deviation(scale, alpha=None):\n",
    "    if alpha is None:\n",
    "        alpha = float(input(\"Enter the alpha (e.g., 0.05): \"))\n",
    "\n",
    "    max_deviation = discrete_laplacian_scale_to_accuracy(scale=scale, alpha=alpha)\n",
    "    return max_deviation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ba023b56",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_log_file(log_file_path):\n",
    "    print(\"Trying to read log file:\", log_file_path)\n",
    "    try:\n",
    "        if os.path.exists(log_file_path):\n",
    "            with open(log_file_path, 'r') as log_file:\n",
    "                log_contents = log_file.read()\n",
    "            return log_contents\n",
    "        else:\n",
    "            return f\"Log file '{log_file_path}' not found.\"\n",
    "    except Exception as e:\n",
    "        return f\"Error occurred while reading the log file: {e}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1c61882c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gspread\n",
    "from gspread_dataframe import set_with_dataframe\n",
    "\n",
    "def parse_log_content(log_content):\n",
    "    parsed_data = []\n",
    "    unique_entries = set()\n",
    "\n",
    "    for line in log_content.split(\"\\n\"):\n",
    "        if line.strip() and '-' not in line:\n",
    "            print(f\"Warning: Skipping line due to invalid format: {line}\")\n",
    "            continue  # Skip to the next iteration without processing further\n",
    "        if not line.strip():\n",
    "            continue\n",
    "\n",
    "        try:\n",
    "            if \" - \" not in line:\n",
    "                raise ValueError(f\"Invalid log format in line: {line}\")\n",
    "            # Split based on \"-\"\n",
    "            timestamp_str, event_duration_str = line.split(\" - \", 1)\n",
    "            # Remove trailing white space\n",
    "            timestamp = timestamp_str.strip()\n",
    "            #print(f\"Timestamp: {timestamp_str}\")\n",
    "\n",
    "            # Split the event and duration string with a maximum of 1 split\n",
    "            event, *duration_str_parts = event_duration_str.split(\": \", 1)\n",
    "            # Join the duration string parts back together in case there were additional separators in the event description\n",
    "            duration_str = \": \".join(duration_str_parts).strip()\n",
    "#             print(f\"Event: {event}\")\n",
    "#             print(f\"Event-Duration: {event_duration_str}\")\n",
    "#             print(f\"Duration String Parts: {duration_str_parts}\")\n",
    "\n",
    "            if not duration_str:\n",
    "                raise ValueError(f\"Invalid log format in line: {line}\", \"missing duration string\")\n",
    "\n",
    "            # Check if the duration includes units (e.g., 'ms' or 'seconds')\n",
    "            if 'ms' in duration_str:\n",
    "                duration = float(duration_str.split(\" ms\")[0])\n",
    "            elif 'seconds' in duration_str:\n",
    "                duration = float(duration_str.split(\" seconds\")[0]) * 1000  # Convert seconds to milliseconds\n",
    "            else:\n",
    "                raise ValueError(f\"Invalid duration format in line: {line}\", \"missing time\")\n",
    "\n",
    "            # Extract the value of 'k' from the event description\n",
    "            k_str_parts = event.split(\"k=\")\n",
    "            if len(k_str_parts) < 2:\n",
    "                raise ValueError(f\"Invalid 'k' value in line: {line}\")\n",
    "            k_str = k_str_parts[1].split(\">\")[0]\n",
    "            k = float(k_str)\n",
    "\n",
    "            # Extract the data size from the event description\n",
    "            data_size_parts = event.split(\"<Datasize=\")\n",
    "            if len(data_size_parts) < 2:\n",
    "                raise ValueError(f\"Invalid 'Data Size' value in line: {line}\")\n",
    "            data_size = data_size_parts[1].split(\",\")[0]\n",
    "\n",
    "            # Check if the entry is unique, if yes, add it to the parsed_data list and the unique_entries set\n",
    "            entry_key = (timestamp, event, k, duration, data_size)\n",
    "            if entry_key not in unique_entries:\n",
    "                unique_entries.add(entry_key)\n",
    "                parsed_data.append({\"Timestamp\": timestamp, \"Event\": event, \"k\": k, \"Duration\": duration, \"Datasize\": data_size})\n",
    "            else:\n",
    "                print(f\"Warning: Skipping duplicate entry: {line}\")\n",
    "\n",
    "        except ValueError as e:\n",
    "            # Handle the case when the line does not match the expected format, here its the separator -------\n",
    "            print(f\"Warning for parse_log_content: skip line invalid form: {line}\")\n",
    "#             print(f\"Timestamp: {timestamp_str}\")\n",
    "#             print(f\"Event-Duration: {event_duration_str}\")\n",
    "#             print(f\"Event: {event}\")\n",
    "#             print(f\"Duration String Parts: {duration_str_parts}\")\n",
    "\n",
    "    return parsed_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a88a50e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import warnings\n",
    "import re\n",
    "def write_log_GS(parsed_data):\n",
    "    if not parsed_data:\n",
    "        print(\"No valid log lines were found.\")\n",
    "        return\n",
    "    try:\n",
    "        # Authenticate with Google Sheets using the credentials JSON file\n",
    "        gc = gspread.service_account(filename='verse-project-timesheet-f4792514dd22.json')\n",
    "        sheet = gc.open('Code Execution Time Log')\n",
    "        worksheet = sheet.worksheet('Sheet1')\n",
    "\n",
    "        # Create a list of column names\n",
    "        column_names = list(parsed_data[0].keys())\n",
    "\n",
    "        # Clear the existing data in the worksheet and write the column names\n",
    "        worksheet.clear()\n",
    "        worksheet.append_row(column_names)\n",
    "\n",
    "        # Convert the parsed data to a list of lists (values) for writing to the Google Sheet\n",
    "        values = [list(entry.values()) for entry in parsed_data]\n",
    "\n",
    "        # Write the values to the Google Sheet, starting from cell A2 (after the column names)\n",
    "        with warnings.catch_warnings():\n",
    "            warnings.simplefilter(\"ignore\")\n",
    "            worksheet.update('A2', values, value_input_option='USER_ENTERED')\n",
    "\n",
    "        print(\"Log data has been written to the Google Sheet.\")\n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred while writing data to Google Sheet: {e}\")\n",
    "        print(\"Error details:\")\n",
    "        import traceback\n",
    "        traceback.print_exc()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2f3059d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_execution_time(event_data, data_size):\n",
    "    for event, k, time_ms in event_data:\n",
    "        logging.info(f'{event} <Datasize={data_size},k={k}>: {time_ms:.3f} ms')\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9a52e50a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gspread\n",
    "from oauth2client.service_account import ServiceAccountCredentials\n",
    "\n",
    "def read_data_from_google_sheets(sheet_name, worksheet_name):\n",
    "    try:\n",
    "        # Authenticate with Google Sheets using the credentials JSON file\n",
    "        scope = ['https://spreadsheets.google.com/feeds', 'https://www.googleapis.com/auth/drive']\n",
    "        creds = ServiceAccountCredentials.from_json_keyfile_name('verse-project-timesheet-f4792514dd22.json', scope)\n",
    "        client = gspread.authorize(creds)\n",
    "\n",
    "        # Open the Google Sheet\n",
    "        sheet = client.open(sheet_name)\n",
    "\n",
    "        # Read data from the specified worksheet\n",
    "        worksheet = sheet.worksheet(worksheet_name)\n",
    "        data = worksheet.get_all_records()\n",
    "\n",
    "        return data\n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred while reading data from Google Sheet: {e}\")\n",
    "        return None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4284869b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_sizes = ['1KB', '500KB', '1MB', '500MB', '1GB']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f36fbe42",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Access Google Sheet at this link:\n",
      "https://docs.google.com/spreadsheets/d/12uLBDyk8io9BvxWPqtluN-K1pmnGekNdFz2TmgqoaDM/edit?usp=sharing\n",
      "DATASIZE: 1KB\n",
      "Dummy Percentage <for DP Private only>: 25.0\n",
      "Trying to read log file: /Users/anhpham/Projects/verse_project/code_execution.log\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:35 - --------------------------------------\n",
      "Log data has been written to the Google Sheet.\n",
      "Data read from Google Sheet\n",
      " [{'Timestamp': '2023-08-01 0:08:35', 'Event': 'Scan <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 2.574, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Filter <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 1.381, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Pad <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.817, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Fully Public <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.001, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Fully Private <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.15, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'DP Private <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.121, 'Datasize': '1KB'}]\n",
      "\n",
      "DATASIZE: 500KB\n",
      "Dummy Percentage <for DP Private only>: 25.0\n",
      "Trying to read log file: /Users/anhpham/Projects/verse_project/code_execution.log\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:35 - --------------------------------------\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:38 - --------------------------------------\n",
      "Log data has been written to the Google Sheet.\n",
      "Data read from Google Sheet\n",
      " [{'Timestamp': '2023-08-01 0:08:35', 'Event': 'Scan <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 2.574, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Filter <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 1.381, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Pad <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.817, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Fully Public <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.001, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Fully Private <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.15, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'DP Private <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.121, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Scan <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 28.165, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Filter <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 2.262, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Pad <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 4.617, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Fully Public <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.003, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Fully Private <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.265, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'DP Private <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.179, 'Datasize': '500KB'}]\n",
      "\n",
      "DATASIZE: 1MB\n",
      "Dummy Percentage <for DP Private only>: 25.0\n",
      "Trying to read log file: /Users/anhpham/Projects/verse_project/code_execution.log\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:35 - --------------------------------------\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:38 - --------------------------------------\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:40 - --------------------------------------\n",
      "Log data has been written to the Google Sheet.\n",
      "Data read from Google Sheet\n",
      " [{'Timestamp': '2023-08-01 0:08:35', 'Event': 'Scan <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 2.574, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Filter <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 1.381, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Pad <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.817, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Fully Public <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.001, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Fully Private <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.15, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'DP Private <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.121, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Scan <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 28.165, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Filter <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 2.262, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Pad <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 4.617, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Fully Public <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.003, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Fully Private <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.265, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'DP Private <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.179, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Scan <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 49.716, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Filter <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 2.404, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Pad <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 6.49, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Fully Public <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 0.002, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Fully Private <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 0.245, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'DP Private <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 0.164, 'Datasize': '1MB'}]\n",
      "\n",
      "DATASIZE: 500MB\n",
      "Dummy Percentage <for DP Private only>: 25.0\n",
      "Trying to read log file: /Users/anhpham/Projects/verse_project/code_execution.log\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:35 - --------------------------------------\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:38 - --------------------------------------\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:40 - --------------------------------------\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:52 - --------------------------------------\n",
      "Log data has been written to the Google Sheet.\n",
      "Data read from Google Sheet\n",
      " [{'Timestamp': '2023-08-01 0:08:35', 'Event': 'Scan <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 2.574, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Filter <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 1.381, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Pad <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.817, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Fully Public <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.001, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Fully Private <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.15, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'DP Private <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.121, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Scan <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 28.165, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Filter <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 2.262, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Pad <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 4.617, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Fully Public <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.003, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Fully Private <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.265, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'DP Private <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.179, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Scan <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 49.716, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Filter <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 2.404, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Pad <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 6.49, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Fully Public <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 0.002, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Fully Private <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 0.245, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'DP Private <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 0.164, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'Scan <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 7929.016, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'Filter <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 115.753, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'Pad <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 1362.412, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'Fully Public <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 0.003, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'Fully Private <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 3.436, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'DP Private <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 1.785, 'Datasize': '500MB'}]\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DATASIZE: 1GB\n",
      "Dummy Percentage <for DP Private only>: 25.0\n",
      "Trying to read log file: /Users/anhpham/Projects/verse_project/code_execution.log\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:35 - --------------------------------------\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:38 - --------------------------------------\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:40 - --------------------------------------\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:08:52 - --------------------------------------\n",
      "Warning for parse_log_content: skip line invalid form: 2023-08-01 00:09:14 - --------------------------------------\n",
      "Log data has been written to the Google Sheet.\n",
      "Data read from Google Sheet\n",
      " [{'Timestamp': '2023-08-01 0:08:35', 'Event': 'Scan <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 2.574, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Filter <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 1.381, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Pad <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.817, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Fully Public <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.001, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'Fully Private <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.15, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:35', 'Event': 'DP Private <Datasize=1KB,k=0.25>', 'k': 0.25, 'Duration': 0.121, 'Datasize': '1KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Scan <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 28.165, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Filter <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 2.262, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Pad <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 4.617, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Fully Public <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.003, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'Fully Private <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.265, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:38', 'Event': 'DP Private <Datasize=500KB,k=0.25>', 'k': 0.25, 'Duration': 0.179, 'Datasize': '500KB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Scan <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 49.716, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Filter <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 2.404, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Pad <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 6.49, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Fully Public <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 0.002, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'Fully Private <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 0.245, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:40', 'Event': 'DP Private <Datasize=1MB,k=0.25>', 'k': 0.25, 'Duration': 0.164, 'Datasize': '1MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'Scan <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 7929.016, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'Filter <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 115.753, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'Pad <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 1362.412, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'Fully Public <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 0.003, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'Fully Private <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 3.436, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:08:52', 'Event': 'DP Private <Datasize=500MB,k=0.25>', 'k': 0.25, 'Duration': 1.785, 'Datasize': '500MB'}, {'Timestamp': '2023-08-01 0:09:14', 'Event': 'Scan <Datasize=1GB,k=0.25>', 'k': 0.25, 'Duration': 16260.006, 'Datasize': '1GB'}, {'Timestamp': '2023-08-01 0:09:14', 'Event': 'Filter <Datasize=1GB,k=0.25>', 'k': 0.25, 'Duration': 255.101, 'Datasize': '1GB'}, {'Timestamp': '2023-08-01 0:09:14', 'Event': 'Pad <Datasize=1GB,k=0.25>', 'k': 0.25, 'Duration': 2740.883, 'Datasize': '1GB'}, {'Timestamp': '2023-08-01 0:09:14', 'Event': 'Fully Public <Datasize=1GB,k=0.25>', 'k': 0.25, 'Duration': 0.014, 'Datasize': '1GB'}, {'Timestamp': '2023-08-01 0:09:14', 'Event': 'Fully Private <Datasize=1GB,k=0.25>', 'k': 0.25, 'Duration': 6.188, 'Datasize': '1GB'}, {'Timestamp': '2023-08-01 0:09:14', 'Event': 'DP Private <Datasize=1GB,k=0.25>', 'k': 0.25, 'Duration': 2.249, 'Datasize': '1GB'}]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def main(data_sizes):\n",
    "          \n",
    "    #start writing csv code here without interfering with postgress code   \n",
    "    #'''\n",
    "    try:\n",
    "        # Initialize the log file\n",
    "        log_file_path = os.path.join(os.getcwd(), 'code_execution.log')\n",
    "        logging.basicConfig(filename=log_file_path, level=logging.INFO,\n",
    "                            format='%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')\n",
    "        print(\"\"\"Access Google Sheet at this link:\n",
    "https://docs.google.com/spreadsheets/d/12uLBDyk8io9BvxWPqtluN-K1pmnGekNdFz2TmgqoaDM/edit?usp=sharing\"\"\")\n",
    " \n",
    "        for data_size in data_sizes:\n",
    "            # Scan data\n",
    "            start_scan = time.time()\n",
    "            df = csv_scan(f\"{data_size}.csv\")\n",
    "            end_scan = time.time()\n",
    "            scan_time = (end_scan - start_scan) * 10**3\n",
    "\n",
    "            # Filter data\n",
    "            start_filter = time.time()\n",
    "            filtered_column = 'Age'\n",
    "            condition = f\"{filtered_column} < 26\"\n",
    "            young_df = filter_df(df, condition)\n",
    "            end_filter = time.time()\n",
    "            filter_time = (end_filter - start_filter) * 10**3\n",
    "\n",
    "            # Pad data\n",
    "            start_pad = time.time()\n",
    "            young_padded = pad_data(df, young_df, \"dummy\")\n",
    "            end_pad = time.time()\n",
    "            pad_time = (end_pad - start_pad) * 10**3\n",
    "\n",
    "            # Fully Public count\n",
    "            start_public = time.time()\n",
    "            young_count = young_df.shape[0]\n",
    "            end_public = time.time()\n",
    "            public_time = (end_public - start_public) * 10**3\n",
    "\n",
    "            # Fully Private count\n",
    "            start_priv = time.time()\n",
    "            real_count = count_real(young_padded, 'dummy')\n",
    "            end_priv = time.time()\n",
    "            full_priv_time = (end_priv - start_priv) * 10**3\n",
    "\n",
    "            # DP Private pad\n",
    "            k = 0.25\n",
    "            dp_padded = DP_pad_data(young_df, df, k, \"dummy\")\n",
    "            start_dp = time.time()\n",
    "            dp_padded_count = count_real(dp_padded, 'dummy')\n",
    "            end_dp = time.time()\n",
    "            dp_padded_time = (end_dp - start_dp) * 10**3\n",
    "            \n",
    "            # print the difference between start and end time in milli. secs\n",
    "            # print(\"TIME to 3 decimal \") \n",
    "            print(\"DATASIZE:\", data_size)\n",
    "            print(\"Dummy Percentage <for DP Private only>:\", k*100)\n",
    "            # Log the time duration of each event\n",
    "            events = [\n",
    "                ('Scan', k, scan_time),\n",
    "                ('Filter', k, filter_time),\n",
    "                ('Pad', k, pad_time),\n",
    "                ('Fully Public', k, public_time),\n",
    "                ('Fully Private', k, full_priv_time),\n",
    "                ('DP Private', k, dp_padded_time)\n",
    "            ]\n",
    "\n",
    "\n",
    "            # logging the time duration of each event\n",
    "            log_execution_time(events, data_size)\n",
    "            logging.info('-' * 38)\n",
    "\n",
    "            # Open and read log\n",
    "            #log_file_path = 'code_execution.log'\n",
    "            log_content = read_log_file(log_file_path)\n",
    "            #print(f\"LOG CONTENT\\n{log_content}\")\n",
    "            \n",
    "            # Parse the log content to a structured format\n",
    "            parsed_data = parse_log_content(log_content)\n",
    "            \n",
    "\n",
    "            # Call the write_log_GS function with the parsed data\n",
    "            write_log_GS(parsed_data)\n",
    "\n",
    "#             #Print the log data for verification\n",
    "#             print(\"Log Data After Write:\")\n",
    "#             for entry in parsed_data:\n",
    "#                 print(\"PARSED DATA\\n\", entry)\n",
    "                \n",
    "            data = read_data_from_google_sheets(\"Code Execution Time Log\", \"Sheet1\")\n",
    "            if data:\n",
    "                print(\"Data read from Google Sheet\\n\",data)\n",
    "                #pass\n",
    "                print()\n",
    "            else:\n",
    "                print(\"Failed to read data from Google Sheet.\")\n",
    "            \n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred: {e}\")\n",
    "        print(\"Error details:\")\n",
    "        import traceback\n",
    "        traceback.print_exc()\n",
    "            \n",
    "    #'''\n",
    "            \n",
    "    \n",
    "       \n",
    "if __name__ == \"__main__\":\n",
    "    main(data_sizes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "75363bfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#write some small program to increase file size\n",
    "#to get data points for the x-bar\n",
    "\n",
    "\n",
    "#scan, filter, join, count\n",
    "#extract \n",
    "#run individually\n",
    "#just scan, just joint\n",
    "#if 25% 50% percent dummy then how the performance change\n",
    "#setting 25% dummy\n",
    "\n",
    "\n",
    "#mechanism - getscale\n",
    "#check DP pad\n",
    "#about logging, check if my log is checking each stage? is that what we re trying to do\n",
    "#try putting log in google sheet\n",
    "#visualization\n",
    "\n",
    "#iguring out how to extract datasize to become data points on x-axis\n",
    "#formulate problem, we need an extra column for datasize, how do we write this datasize column in the function\n",
    "#but handle cases when this datasize is not applicable as well\n",
    "\n",
    "\n",
    "#commenting the example code\n",
    "#subsitute"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}