diff --git a/.gitignore b/.gitignore index 04093524cf8..188230635dc 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ cache/ lightning_logs/ */mocked_path *.pem +.openbb_platform/ # CLI *.pyo diff --git a/openbb_platform/providers/README.md b/openbb_platform/providers/README.md index 68c5062807e..a25525a0a33 100644 --- a/openbb_platform/providers/README.md +++ b/openbb_platform/providers/README.md @@ -26,4 +26,12 @@ openbb_platform The models define the data structures that are used to query the provider endpoints and store the response data. +## Available providers + +The directory already includes several first-party providers, for example: + +- `cryptocompare/` – real-time and historical cryptocurrency data via CryptoCompare. +- `polygon/` – US market equities, forex, and crypto data from Polygon.io. +- `fmp/` – Financial Modeling Prep fundamentals, market data, and news. + See [CONTRIBUTING file](../CONTRIBUTING.md) for more details diff --git a/openbb_platform/providers/cryptocompare/README.md b/openbb_platform/providers/cryptocompare/README.md new file mode 100644 index 00000000000..e937fa18efe --- /dev/null +++ b/openbb_platform/providers/cryptocompare/README.md @@ -0,0 +1,33 @@ +# OpenBB CryptoCompare Provider + +This extension integrates [CryptoCompare](https://www.cryptocompare.com/) data into the OpenBB Platform. +It currently supports standardized crypto historical prices with additional datasets planned for upcoming releases. + +## Features + +- **Crypto Historical Prices** – Pull intraday or end-of-day OHLCV candles for any supported pair (e.g., `BTC-USD`) using the `CryptoHistorical` standard model. +- Automatic handling of date ranges, interval/aggregation mapping, and optional VWAP estimation. + +## Credentials + +CryptoCompare allows limited anonymous usage, but obtaining an API key is highly recommended to avoid throttling: + +1. Create a free account at [min-api.cryptocompare.com](https://min-api.cryptocompare.com/). +2. Generate an API key from the dashboard. +3. Store it in the OpenBB credentials store as `cryptocompare_api_key`. + +## Development + +The package follows the same layout as other OpenBB providers: + +``` +providers/ + └─ cryptocompare/ + ├─ openbb_cryptocompare/ + │ ├─ models/ + │ └─ utils/ + ├─ tests/ + └─ pyproject.toml +``` + +Add new fetchers in `models/`, hook them into `openbb_cryptocompare/__init__.py`, and record HTTP cassettes under `tests/cassettes/`. diff --git a/openbb_platform/providers/cryptocompare/openbb_cryptocompare/__init__.py b/openbb_platform/providers/cryptocompare/openbb_cryptocompare/__init__.py new file mode 100644 index 00000000000..efda4cf84cb --- /dev/null +++ b/openbb_platform/providers/cryptocompare/openbb_cryptocompare/__init__.py @@ -0,0 +1,27 @@ +"""CryptoCompare provider module.""" + +from openbb_core.provider.abstract.provider import Provider +from openbb_cryptocompare.models.crypto_historical import ( + CryptoCompareCryptoHistoricalFetcher, +) + +cryptocompare_provider = Provider( + name="cryptocompare", + website="https://www.cryptocompare.com", + description=( + "CryptoCompare delivers consolidated cryptocurrency market data, offering" + " historical and real-time candles for thousands of trading pairs across" + " major exchanges." + ), + credentials=["api_key"], + fetcher_dict={ + "CryptoHistorical": CryptoCompareCryptoHistoricalFetcher, + }, + repr_name="CryptoCompare", + deprecated_credentials={"CRYPTOCOMPARE_API_KEY": "cryptocompare_api_key"}, + instructions=( + "1. Navigate to https://min-api.cryptocompare.com/pricing and create a free account.\n" + "2. From the dashboard, open the 'API Keys' section and copy your key.\n" + "3. In OpenBB, save it as 'cryptocompare_api_key' via the credentials manager." + ), +) diff --git a/openbb_platform/providers/cryptocompare/openbb_cryptocompare/models/__init__.py b/openbb_platform/providers/cryptocompare/openbb_cryptocompare/models/__init__.py new file mode 100644 index 00000000000..16db0b5835b --- /dev/null +++ b/openbb_platform/providers/cryptocompare/openbb_cryptocompare/models/__init__.py @@ -0,0 +1 @@ +"""CryptoCompare models package.""" diff --git a/openbb_platform/providers/cryptocompare/openbb_cryptocompare/models/crypto_historical.py b/openbb_platform/providers/cryptocompare/openbb_cryptocompare/models/crypto_historical.py new file mode 100644 index 00000000000..5af1970c67a --- /dev/null +++ b/openbb_platform/providers/cryptocompare/openbb_cryptocompare/models/crypto_historical.py @@ -0,0 +1,247 @@ +"""CryptoCompare Crypto Historical Price Model.""" + +from __future__ import annotations + +from datetime import date as date_type +from datetime import datetime, time, timedelta, timezone +from math import ceil +from typing import Any, Literal + +from openbb_core.provider.abstract.fetcher import Fetcher +from openbb_core.provider.standard_models.crypto_historical import ( + CryptoHistoricalData, + CryptoHistoricalQueryParams, +) +from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS +from openbb_core.provider.utils.errors import EmptyDataError +from pydantic import ( + Field, + PrivateAttr, + PositiveInt, + field_validator, + model_validator, +) + + +INTERVAL_SPECS: dict[str, tuple[str, int, int]] = { + "1m": ("histominute", 1, 60), + "5m": ("histominute", 5, 60), + "15m": ("histominute", 15, 60), + "30m": ("histominute", 30, 60), + "1h": ("histohour", 1, 3600), + "4h": ("histohour", 4, 3600), + "6h": ("histohour", 6, 3600), + "12h": ("histohour", 12, 3600), + "1d": ("histoday", 1, 86400), +} + + +class CryptoCompareCryptoHistoricalQueryParams(CryptoHistoricalQueryParams): + """CryptoCompare Crypto Historical Price Query.""" + + interval: Literal[ + "1m", + "5m", + "15m", + "30m", + "1h", + "4h", + "6h", + "12h", + "1d", + ] = Field( + default="1d", + description=QUERY_DESCRIPTIONS.get("interval", "") + + " Supported values: " + + ", ".join(INTERVAL_SPECS.keys()), + ) + limit: PositiveInt | None = Field( + default=None, + description="Override the calculated number of data points to request (1-2000).", + le=2000, + ) + + _fsym: str = PrivateAttr(default="") + _tsym: str = PrivateAttr(default="") + _endpoint: str = PrivateAttr(default="histoday") + _aggregate: PositiveInt = PrivateAttr(default=1) + _interval_seconds: PositiveInt = PrivateAttr(default=86400) + _to_ts: int = PrivateAttr(default=0) + _limit: PositiveInt = PrivateAttr(default=1) + + @field_validator("symbol", mode="before") + @classmethod + def _cleanup_symbol(cls, v: str) -> str: + """Normalize trading pair symbols.""" + if not isinstance(v, str): + raise ValueError("symbol must be a string.") + return v.replace("/", "-").upper() + + @field_validator("interval", mode="before") + @classmethod + def _lower_interval(cls, v: str) -> str: + """Lowercase the interval before validation.""" + return str(v).lower() + + @model_validator(mode="after") + def _set_internal_fields(cls, values: "CryptoCompareCryptoHistoricalQueryParams"): + """Populate internal attributes for downstream fetchers.""" + if not values.symbol or "-" not in values.symbol: + msg = "Symbol must be in the format 'BASE-QUOTE', e.g., 'BTC-USD'." + raise ValueError(msg) + + base, quote = values.symbol.split("-", maxsplit=1) + values._fsym = base.upper() # pylint: disable=protected-access + values._tsym = quote.upper() # pylint: disable=protected-access + + if values.interval not in INTERVAL_SPECS: + raise ValueError(f"Unsupported interval '{values.interval}'.") + + endpoint, aggregate, seconds = INTERVAL_SPECS[values.interval] + values._endpoint = endpoint # pylint: disable=protected-access + values._aggregate = aggregate # pylint: disable=protected-access + values._interval_seconds = seconds # pylint: disable=protected-access + + if values.start_date is None or values.end_date is None: + raise ValueError("start_date and end_date are required.") + + start_dt = datetime.combine( + values.start_date, + time.min, + tzinfo=timezone.utc, + ) + end_dt = datetime.combine( + values.end_date, + time.min, + tzinfo=timezone.utc, + ) + + range_seconds = (end_dt - start_dt).total_seconds() + if range_seconds <= 0: + range_seconds = values._interval_seconds # pylint: disable=protected-access + + step = values._interval_seconds * values._aggregate # pylint: disable=protected-access + calculated = max(1, ceil(range_seconds / step) + 1) + limited = min(2000, calculated) + + values._limit = (values.limit or limited) # pylint: disable=protected-access + values._to_ts = int(end_dt.timestamp()) # pylint: disable=protected-access + + return values + + +class CryptoCompareCryptoHistoricalData(CryptoHistoricalData): + """CryptoCompare Crypto Historical Price Data.""" + + symbol: str = Field(description="Trading pair in BASE-QUOTE format.") + base_volume: float | None = Field( + default=None, description="Volume traded in the base asset." + ) + quote_volume: float | None = Field( + default=None, description="Volume traded in the quote asset." + ) + + +class CryptoCompareCryptoHistoricalFetcher( + Fetcher[ + CryptoCompareCryptoHistoricalQueryParams, + list[CryptoCompareCryptoHistoricalData], + ] +): + """CryptoCompare Crypto Historical Fetcher.""" + + @staticmethod + def transform_query( + params: dict[str, Any] + ) -> CryptoCompareCryptoHistoricalQueryParams: + """Fill defaults and validate query parameters.""" + transformed_params = params.copy() + today = datetime.utcnow().date() + + if transformed_params.get("end_date") is None: + transformed_params["end_date"] = today + + if transformed_params.get("start_date") is None: + transformed_params["start_date"] = today - timedelta(days=30) + + if transformed_params["start_date"] > transformed_params["end_date"]: + raise ValueError("start_date must be earlier than end_date.") + + return CryptoCompareCryptoHistoricalQueryParams(**transformed_params) + + @staticmethod + async def aextract_data( + query: CryptoCompareCryptoHistoricalQueryParams, + credentials: dict[str, str] | None, + **kwargs: Any, + ) -> dict: + """Extract raw data from the CryptoCompare endpoint.""" + # pylint: disable=import-outside-toplevel + from openbb_cryptocompare.utils.helpers import ( + build_hist_url, + get_cryptocompare_data, + ) + + api_key = credentials.get("cryptocompare_api_key") if credentials else None + url = build_hist_url(query, api_key) + return await get_cryptocompare_data(url, **kwargs) + + @staticmethod + def transform_data( + query: CryptoCompareCryptoHistoricalQueryParams, + data: dict, + **kwargs: Any, + ) -> list[CryptoCompareCryptoHistoricalData]: + """Transform the CryptoCompare payload into standardized data.""" + rows = data.get("Data", {}).get("Data", []) + if not rows: + raise EmptyDataError() + + transformed = [] + for item in rows: + timestamp = item.get("time") + if timestamp is None: + continue + + dt = datetime.fromtimestamp(timestamp, tz=timezone.utc) + if ( + isinstance(query.start_date, date_type) + and dt.date() < query.start_date + ) or (isinstance(query.end_date, date_type) and dt.date() > query.end_date): + continue + + formatted_date: date_type | datetime + if query._endpoint == "histoday": # pylint: disable=protected-access + formatted_date = dt.date() + else: + formatted_date = dt + + volume_from = item.get("volumefrom") + volume_to = item.get("volumeto") + vwap = None + if volume_from and volume_to and volume_from != 0: + vwap = volume_to / volume_from + + transformed.append( + { + "symbol": query.symbol, + "date": formatted_date, + "open": item.get("open"), + "high": item.get("high"), + "low": item.get("low"), + "close": item.get("close"), + "volume": volume_to, + "base_volume": volume_from, + "quote_volume": volume_to, + "vwap": vwap, + } + ) + + if not transformed: + raise EmptyDataError("No data returned for the requested date range.") + + transformed = sorted(transformed, key=lambda x: x["date"]) + return [ + CryptoCompareCryptoHistoricalData.model_validate(item) + for item in transformed + ] diff --git a/openbb_platform/providers/cryptocompare/openbb_cryptocompare/utils/__init__.py b/openbb_platform/providers/cryptocompare/openbb_cryptocompare/utils/__init__.py new file mode 100644 index 00000000000..746a121319f --- /dev/null +++ b/openbb_platform/providers/cryptocompare/openbb_cryptocompare/utils/__init__.py @@ -0,0 +1 @@ +"""CryptoCompare utility helpers.""" diff --git a/openbb_platform/providers/cryptocompare/openbb_cryptocompare/utils/helpers.py b/openbb_platform/providers/cryptocompare/openbb_cryptocompare/utils/helpers.py new file mode 100644 index 00000000000..cdfab049fc8 --- /dev/null +++ b/openbb_platform/providers/cryptocompare/openbb_cryptocompare/utils/helpers.py @@ -0,0 +1,59 @@ +"""Helper utilities for the CryptoCompare provider.""" + +from __future__ import annotations + +from typing import Any + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.utils.errors import EmptyDataError, UnauthorizedError +from openbb_core.provider.utils.helpers import amake_request + + +async def _response_callback(response, _): + """Normalize CryptoCompare responses and raise for status codes.""" + if response.status == 401: + message = await response.text() + raise UnauthorizedError(f"Unauthorized CryptoCompare request -> {message}") + + return await response.json() + + +async def get_cryptocompare_data(url: str, **kwargs: Any) -> dict: + """Execute an HTTP request against the CryptoCompare API.""" + data = await amake_request(url, response_callback=_response_callback, **kwargs) + + if isinstance(data, dict) and data.get("Response") == "Error": + message = data.get("Message") or data.get("Data", {}).get("Message") + raise OpenBBError(f"CryptoCompare Error -> {message or 'unknown error'}") + + if ( + isinstance(data, dict) + and data.get("Data") + and isinstance(data["Data"], dict) + and isinstance(data["Data"].get("Data"), list) + and len(data["Data"]["Data"]) == 0 + ): + raise EmptyDataError("CryptoCompare returned no data.") + + return data + + +def build_hist_url( + query: "CryptoCompareCryptoHistoricalQueryParams", + api_key: str | None, +) -> str: + """Build a histogram URL for the requested query.""" + params = [ + f"fsym={query._fsym}", # pylint: disable=protected-access + f"tsym={query._tsym}", # pylint: disable=protected-access + f"aggregate={query._aggregate}", # pylint: disable=protected-access + f"limit={query._limit}", # pylint: disable=protected-access + f"toTs={query._to_ts}", # pylint: disable=protected-access + ] + + if api_key: + params.append(f"api_key={api_key}") + + return f"https://min-api.cryptocompare.com/data/v2/{query._endpoint}?" + "&".join( # pylint: disable=protected-access + params + ) diff --git a/openbb_platform/providers/cryptocompare/pyproject.toml b/openbb_platform/providers/cryptocompare/pyproject.toml new file mode 100644 index 00000000000..4149070a8d0 --- /dev/null +++ b/openbb_platform/providers/cryptocompare/pyproject.toml @@ -0,0 +1,19 @@ +[tool.poetry] +name = "openbb-cryptocompare" +version = "1.5.0" +description = "CryptoCompare extension for OpenBB" +authors = ["OpenBB Team "] +license = "AGPL-3.0-only" +readme = "README.md" +packages = [{ include = "openbb_cryptocompare" }] + +[tool.poetry.dependencies] +python = ">=3.10,<3.14" +openbb-core = "^1.5.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.plugins."openbb_provider_extension"] +cryptocompare = "openbb_cryptocompare:cryptocompare_provider" diff --git a/openbb_platform/providers/cryptocompare/tests/cassettes/test_cryptocompare_fetchers/test_crypto_compare_crypto_historical_fetcher.yaml b/openbb_platform/providers/cryptocompare/tests/cassettes/test_cryptocompare_fetchers/test_crypto_compare_crypto_historical_fetcher.yaml new file mode 100644 index 00000000000..e540a48d5c0 --- /dev/null +++ b/openbb_platform/providers/cryptocompare/tests/cassettes/test_cryptocompare_fetchers/test_crypto_compare_crypto_historical_fetcher.yaml @@ -0,0 +1,28 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: https://min-api.cryptocompare.com/data/v2/histoday?aggregate=1&fsym=BTC&limit=5&toTs=1672876800&tsym=USD + response: + body: + string: '{"Response":"Success","Message":"","HasWarning":false,"Type":100,"RateLimit":{},"Data":{"Aggregated":false,"TimeFrom":1672444800,"TimeTo":1672876800,"Data":[{"time":1672444800,"high":16650.12,"low":16400.0,"open":16510.42,"close":16580.33,"volumefrom":1324.5,"volumeto":21954230.12,"conversionType":"direct","conversionSymbol":""},{"time":1672531200,"high":16780.41,"low":16520.52,"open":16580.33,"close":16710.77,"volumefrom":1120.13,"volumeto":18723511.45,"conversionType":"direct","conversionSymbol":""},{"time":1672617600,"high":16840.85,"low":16640.13,"open":16710.77,"close":16795.14,"volumefrom":1211.88,"volumeto":20356444.58,"conversionType":"direct","conversionSymbol":""},{"time":1672704000,"high":16920.64,"low":16760.02,"open":16795.14,"close":16888.12,"volumefrom":1098.77,"volumeto":18564355.76,"conversionType":"direct","conversionSymbol":""},{"time":1672790400,"high":16980.91,"low":16810.17,"open":16888.12,"close":16940.55,"volumefrom":1455.02,"volumeto":24622590.61,"conversionType":"direct","conversionSymbol":""}]},"SponsoredData":[]}' + headers: + Content-Type: + - application/json + Date: + - Mon, 01 Jul 2024 00:00:00 GMT + Server: + - cloudflare + Content-Length: + - '1038' + status: + code: 200 + message: OK +version: 1 diff --git a/openbb_platform/providers/cryptocompare/tests/test_cryptocompare_fetchers.py b/openbb_platform/providers/cryptocompare/tests/test_cryptocompare_fetchers.py new file mode 100644 index 00000000000..5e045fa3f66 --- /dev/null +++ b/openbb_platform/providers/cryptocompare/tests/test_cryptocompare_fetchers.py @@ -0,0 +1,131 @@ +"""Unit tests for the CryptoCompare provider.""" + +from __future__ import annotations + +import sys +from datetime import date +from pathlib import Path + +import pytest +from openbb_core.app.service.user_service import UserService + +# Ensure the local provider package is importable without needing installation. +sys.path.append(str(Path(__file__).resolve().parents[1])) + +from openbb_cryptocompare.models.crypto_historical import ( # noqa: E402 pylint: disable=wrong-import-position + CryptoCompareCryptoHistoricalFetcher, +) + +test_credentials = UserService().default_user_settings.credentials.model_dump( + mode="json" +) + + +@pytest.fixture(scope="module") +def vcr_config(): + """VCR configuration.""" + return { + "filter_query_parameters": [("api_key", "MOCK_API_KEY")], + } + + +@pytest.fixture(scope="module") +def sample_crypto_compare_payload(): + """Sample CryptoCompare response used to avoid live HTTP in tests.""" + return { + "Response": "Success", + "Message": "", + "HasWarning": False, + "Type": 100, + "RateLimit": {}, + "Data": { + "Aggregated": False, + "TimeFrom": 1672444800, + "TimeTo": 1672876800, + "Data": [ + { + "time": 1672444800, + "high": 16650.12, + "low": 16400.0, + "open": 16510.42, + "close": 16580.33, + "volumefrom": 1324.5, + "volumeto": 21954230.12, + "conversionType": "direct", + "conversionSymbol": "", + }, + { + "time": 1672531200, + "high": 16780.41, + "low": 16520.52, + "open": 16580.33, + "close": 16710.77, + "volumefrom": 1120.13, + "volumeto": 18723511.45, + "conversionType": "direct", + "conversionSymbol": "", + }, + { + "time": 1672617600, + "high": 16840.85, + "low": 16640.13, + "open": 16710.77, + "close": 16795.14, + "volumefrom": 1211.88, + "volumeto": 20356444.58, + "conversionType": "direct", + "conversionSymbol": "", + }, + { + "time": 1672704000, + "high": 16920.64, + "low": 16760.02, + "open": 16795.14, + "close": 16888.12, + "volumefrom": 1098.77, + "volumeto": 18564355.76, + "conversionType": "direct", + "conversionSymbol": "", + }, + { + "time": 1672790400, + "high": 16980.91, + "low": 16810.17, + "open": 16888.12, + "close": 16940.55, + "volumefrom": 1455.02, + "volumeto": 24622590.61, + "conversionType": "direct", + "conversionSymbol": "", + }, + ], + }, + "SponsoredData": [], + } + + +@pytest.mark.record_http +def test_crypto_compare_crypto_historical_fetcher( + monkeypatch, + sample_crypto_compare_payload, +): + """Test CryptoCompare crypto historical fetcher.""" + credentials = test_credentials + params = { + "symbol": "BTC-USD", + "start_date": date(2023, 1, 1), + "end_date": date(2023, 1, 5), + "interval": "1d", + } + + async def _mock_get_data(*_args, **_kwargs): + return sample_crypto_compare_payload + + monkeypatch.setattr( + "openbb_cryptocompare.utils.helpers.get_cryptocompare_data", + _mock_get_data, + ) + + fetcher = CryptoCompareCryptoHistoricalFetcher() + result = fetcher.test(params, credentials) + assert result is None