logs.py

import settings
import os
import pickle
from collections import OrderedDict

if not os.path.exists(settings.SCRAPE_LOG_FILE_PATH):
    data_log = OrderedDict()
    pickle.dump(data_log, open(settings.SCRAPE_LOG_FILE_PATH, "wb"))
if not os.path.exists(settings.EXTRACT_LOG_FILE_PATH):
    data_log = OrderedDict()
    pickle.dump(data_log, open(settings.EXTRACT_LOG_FILE_PATH, "wb"))


def add_scrape_data(symbol, scrape_data, complete):
    """Add data regarding scrape to scrape log."""

    if complete:
        complete_key = 'complete'
    else:
        complete_key = 'incomplete'
    data_log = pickle.load(open(settings.SCRAPE_LOG_FILE_PATH, "rb"))
    try:
        data_log[symbol]
        data_log[symbol][complete_key] = scrape_data
    except KeyError:
        data_log[symbol] = {}
        data_log[symbol]['complete'] = None
        data_log[symbol]['incomplete'] = None
        data_log[symbol][complete_key] = scrape_data
    pickle.dump(data_log, open(settings.SCRAPE_LOG_FILE_PATH, "wb"))


def add_extract_data(symbol, extract_data, complete):
    """Add data regarding scrape or extract to master log."""

    if complete:
        complete_key = 'complete'
    else:
        complete_key = 'incomplete'
    data_log = pickle.load(open(settings.EXTRACT_LOG_FILE_PATH, "rb"))
    try:
        data_log[symbol]
        data_log[symbol][complete_key].append(extract_data)
    except KeyError:
        data_log[symbol] = {}
        data_log[symbol]['complete'] = []
        data_log[symbol]['incomplete'] = []
        data_log[symbol][complete_key].append(extract_data)
    pickle.dump(data_log, open(settings.EXTRACT_LOG_FILE_PATH, "wb"))


def check_if_extracted(symbol, date):
    """Check if symbol's date was previously extracted."""

    data_log = pickle.load(open(settings.EXTRACT_LOG_FILE_PATH, "rb"))
    try:
        extracted = data_log[symbol]['complete']
        not_extracted = data_log[symbol]['incomplete']
    except KeyError:
        return False
    if date in extracted:
        return True
    if date in not_extracted:
        return True
    return False