Skip to content

Commit

Permalink
style: moved constants to serps/constants.py
Browse files Browse the repository at this point in the history
  • Loading branch information
mjgiblett committed Sep 30, 2024
1 parent ddce6eb commit 6db48e6
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 17 deletions.
13 changes: 1 addition & 12 deletions serps/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,7 @@

import yaml

from serps.constants import API_PASSWORD, API_USERNAME, QUERIES_PATH, RESULTS_PATH

USER_CONFIG_PATH = Path(os.path.expanduser("~/.config/serps/serpsrc"))

DOCUMENT_DIR_NAME = "SERP Scraper"

DEFAULT_CONFIG = {
API_USERNAME: None,
API_PASSWORD: None,
RESULTS_PATH: os.path.expanduser(f"~/Documents/{DOCUMENT_DIR_NAME}/Results/"),
QUERIES_PATH: os.path.expanduser(f"~/Documents/{DOCUMENT_DIR_NAME}/Queries/"),
}
from serps.constants import DEFAULT_CONFIG, USER_CONFIG_PATH


def merge_configs(default: dict[str, Any], overwrite: dict[str, Any]) -> dict[str, Any]:
Expand Down
20 changes: 18 additions & 2 deletions serps/constants.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,28 @@
# config keys
from os import path
from pathlib import Path

# user config

API_USERNAME = "oxylabs_api_username"
API_PASSWORD = "oxylabs_api_password"
RESULTS_PATH = "results_path"
QUERIES_PATH = "queries_path"

# oxylabs api options
USER_CONFIG_PATH = Path(path.expanduser("~/.config/serps/serpsrc"))

DOCUMENT_DIR_NAME = "SERP Scraper"

DEFAULT_CONFIG = {
API_USERNAME: None,
API_PASSWORD: None,
RESULTS_PATH: path.expanduser(f"~/Documents/{DOCUMENT_DIR_NAME}/Results/"),
QUERIES_PATH: path.expanduser(f"~/Documents/{DOCUMENT_DIR_NAME}/Queries/"),
}

# oxylabs api
# https://developers.oxylabs.io/scraper-apis/web-scraper-api

REQUEST_URL = "https://realtime.oxylabs.io/v1/queries"
API_OPTION_SOURCE = [
"amazon_product",
"amazon_search",
Expand Down
4 changes: 1 addition & 3 deletions serps/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,7 @@
import yaml
from pandas import DataFrame, ExcelWriter, concat

from serps.constants import DATAFRAME_COLUMNS

REQUEST_URL = "https://realtime.oxylabs.io/v1/queries"
from serps.constants import DATAFRAME_COLUMNS, REQUEST_URL


def mkdirs(file_path: Path | str) -> None:
Expand Down

0 comments on commit 6db48e6

Please sign in to comment.