From ec7ff947076908e5e9cad74485373df9afdfaaf6 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 May 2024 07:20:42 -0700 Subject: [PATCH 1/2] [BugFix] SEC ETF Holdings - Try Catch for RemoteDisconnect Error (#6359) * try catch for RemoteDisconnect error * await sleep * remove not implemented helper --------- Co-authored-by: montezdesousa <79287829+montezdesousa@users.noreply.github.com> --- .../sec/openbb_sec/models/etf_holdings.py | 20 ++++- .../providers/sec/openbb_sec/utils/helpers.py | 88 +------------------ 2 files changed, 18 insertions(+), 90 deletions(-) diff --git a/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py b/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py index 15695e7f02b1..89b65b42d64a 100644 --- a/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py +++ b/openbb_platform/providers/sec/openbb_sec/models/etf_holdings.py @@ -2,6 +2,7 @@ # pylint: disable =[unused-argument,too-many-locals,too-many-branches] +import asyncio from datetime import date as dateType from typing import Any, Dict, List, Optional, Union from warnings import warn @@ -329,9 +330,22 @@ async def aextract_data( **kwargs: Any, ) -> Dict: """Return the raw data from the SEC endpoint.""" - filings = await get_nport_candidates( - symbol=query.symbol, use_cache=query.use_cache - ) + # Implement a retry mechanism in case of RemoteDiconnected Error. + retries = 3 + for i in range(retries): + filings = [] + try: + filings = await get_nport_candidates( + symbol=query.symbol, use_cache=query.use_cache + ) + if filings: + break + except Exception as e: + if i < retries - 1: + warn(f"Error: {e}. Retrying...") + await asyncio.sleep(1) + continue + raise e filing_candidates = pd.DataFrame.from_records(filings) if filing_candidates.empty: raise ValueError(f"No N-Port records found for {query.symbol}.") diff --git a/openbb_platform/providers/sec/openbb_sec/utils/helpers.py b/openbb_platform/providers/sec/openbb_sec/utils/helpers.py index 0f72bcdd5ed1..407a8fc773b4 100644 --- a/openbb_platform/providers/sec/openbb_sec/utils/helpers.py +++ b/openbb_platform/providers/sec/openbb_sec/utils/helpers.py @@ -2,19 +2,16 @@ # pylint: disable =unused-argument -from datetime import timedelta from io import BytesIO from typing import Dict, List, Optional, Union from zipfile import ZipFile import pandas as pd -import requests -import requests_cache from aiohttp_client_cache import SQLiteBackend from aiohttp_client_cache.session import CachedSession from openbb_core.app.utils import get_user_cache_directory from openbb_core.provider.utils.helpers import amake_request, make_request -from openbb_sec.utils.definitions import HEADERS, QUARTERS, SEC_HEADERS, TAXONOMIES +from openbb_sec.utils.definitions import HEADERS, SEC_HEADERS async def sec_callback(response, session): @@ -167,89 +164,6 @@ async def cik_map(cik: Union[str, int], use_cache: bool = True) -> str: return symbol -def get_frame( # pylint: disable =too-many-arguments - year: int, - quarter: Optional[QUARTERS] = None, - taxonomy: TAXONOMIES = "us-gaap", - units: str = "USD", - fact: str = "Revenues", - instantaneous: bool = False, - use_cache: bool = True, -) -> Dict: - """Get a frame of data for a given fact. - - The xbrl/frames API aggregates one fact for each reporting entity - that is last filed that most closely fits the calendrical period requested. - - This API supports for annual, quarterly and instantaneous data: - - https://data.sec.gov/api/xbrl/frames/us-gaap/AccountsPayableCurrent/USD/CY2019Q1I.json - - Where the units of measure specified in the XBRL contains a numerator and a denominator, - these are separated by “-per-” such as “USD-per-shares”. Note that the default unit in XBRL is “pure”. - - CY####Q# for quarterly data (duration 91 days +/- 30 days). - Because company financial calendars can start and end on any month or day and even change in length from quarter to - quarter according to the day of the week, the frame data is assembled by the dates that best align with a calendar - quarter or year. Data users should be mindful different reporting start and end dates for facts contained in a frame. - - Example facts: - Revenues - GrossProfit - CostOfRevenue - DividendsCash - DistributedEarnings - AccountsPayableCurrent - OperatingExpenses - OperatingIncomeLoss - NoninterestIncome - InterestAndDebtExpense - IncomeTaxExpenseBenefit - NetIncomeLoss - - Facts where units are, "shares": - WeightedAverageNumberOfDilutedSharesOutstanding - """ - if fact in ["WeightedAverageNumberOfDilutedSharesOutstanding"]: - units = "shares" - sec_session_frames = requests_cache.CachedSession( - f"{get_user_cache_directory()}/http/sec_frames", expire_after=timedelta(days=2) - ) - url = f"https://data.sec.gov/api/xbrl/frames/{taxonomy}/{fact}/{units}/CY{year}" - - if quarter: - url = url + f"Q{quarter}" - - if instantaneous: - url = url + "I" - url = url + ".json" - r = ( - requests.get(url, headers=HEADERS, timeout=5) - if use_cache is False - else sec_session_frames.get(url, headers=HEADERS, timeout=5) - ) - - if r.status_code != 200: - raise RuntimeError(f"Request failed with status code {r.status_code}") - - response = r.json() - - data = sorted(response["data"], key=lambda x: x["val"], reverse=True) - metadata = { - "frame": response["ccp"], - "tag": response["tag"], - "label": response["label"], - "description": response["description"], - "taxonomy": response["taxonomy"], - "unit": response["uom"], - "count": response["pts"], - } - - results = {"metadata": metadata, "data": data} - - return results - - def get_schema_filelist(query: str = "", url: str = "", use_cache: bool = True) -> List: """Get a list of schema files from the SEC website.""" results: List = [] From 380891de4e23cf4bfdc9cbd0904195ad315eaf75 Mon Sep 17 00:00:00 2001 From: Henrique Joaquim Date: Mon, 6 May 2024 15:49:22 +0100 Subject: [PATCH 2/2] [Feature] Improve `OBBject` Registry (#6364) * registry belongs to the session instead * help message on results * abstract the update_completer a bit and also force the re-link of newly created obbjects so they're immidiatly available * new settings to control the obbject registry * new method to remove a certain index from the stack * using the new flags to control messages on the registyr * fix: add spaces to results --help * Update cli/openbb_cli/controllers/feature_flags_controller.py Co-authored-by: montezdesousa <79287829+montezdesousa@users.noreply.github.com> * rename to settings_controller * typo wrong attr * better messages and actually removing the oldest obbject --------- Co-authored-by: Diogo Sousa Co-authored-by: montezdesousa <79287829+montezdesousa@users.noreply.github.com> --- .../argparse_translator/obbject_registry.py | 37 ++++++++++----- cli/openbb_cli/assets/i18n/en.yml | 2 + cli/openbb_cli/controllers/base_controller.py | 32 +++++++++++++ .../controllers/base_platform_controller.py | 35 ++++++++++----- cli/openbb_cli/controllers/cli_controller.py | 21 ++------- ...s_controller.py => settings_controller.py} | 45 ++++++++++++++++--- cli/openbb_cli/models/settings.py | 6 ++- cli/openbb_cli/session.py | 7 +++ 8 files changed, 139 insertions(+), 46 deletions(-) rename cli/openbb_cli/controllers/{feature_flags_controller.py => settings_controller.py} (86%) diff --git a/cli/openbb_cli/argparse_translator/obbject_registry.py b/cli/openbb_cli/argparse_translator/obbject_registry.py index cb1643747b39..848140062a7a 100644 --- a/cli/openbb_cli/argparse_translator/obbject_registry.py +++ b/cli/openbb_cli/argparse_translator/obbject_registry.py @@ -3,35 +3,43 @@ import json from typing import Dict, List -from openbb_core.app.model.abstract.singleton import SingletonMeta from openbb_core.app.model.obbject import OBBject -class Registry(metaclass=SingletonMeta): +class Registry: + """Registry for OBBjects.""" - obbjects: List[OBBject] = [] + def __init__(self): + """Initialize the registry.""" + self._obbjects: List[OBBject] = [] @staticmethod def _contains_obbject(uuid: str, obbjects: List[OBBject]) -> bool: """Check if obbject with uuid is in the registry.""" return any(obbject.id == uuid for obbject in obbjects) - @classmethod - def register(cls, obbject: OBBject): + def register(self, obbject: OBBject): """Designed to add an OBBject instance to the registry.""" - if isinstance(obbject, OBBject) and not cls._contains_obbject( - obbject.id, cls.obbjects + if isinstance(obbject, OBBject) and not self._contains_obbject( + obbject.id, self._obbjects ): - cls.obbjects.append(obbject) + self._obbjects.append(obbject) - @classmethod - def get(cls, idx: int) -> OBBject: + def get(self, idx: int) -> OBBject: """Return the obbject at index idx.""" # the list should work as a stack # i.e., the last element needs to be accessed by idx=0 and so on - reversed_list = list(reversed(cls.obbjects)) + reversed_list = list(reversed(self._obbjects)) return reversed_list[idx] + def remove(self, idx: int = -1): + """Remove the obbject at index idx, default is the last element.""" + # the list should work as a stack + # i.e., the last element needs to be accessed by idx=0 and so on + reversed_list = list(reversed(self._obbjects)) + del reversed_list[idx] + self._obbjects = list(reversed(reversed_list)) + @property def all(self) -> Dict[int, Dict]: """Return all obbjects in the registry""" @@ -65,7 +73,7 @@ def _handle_data_repr(obbject: OBBject) -> str: return data_repr obbjects = {} - for i, obbject in enumerate(list(reversed(self.obbjects))): + for i, obbject in enumerate(list(reversed(self._obbjects))): obbjects[i] = { "route": obbject._route, # pylint: disable=protected-access "provider": obbject.provider, @@ -74,3 +82,8 @@ def _handle_data_repr(obbject: OBBject) -> str: } return obbjects + + @property + def obbjects(self) -> List[OBBject]: + """Return all obbjects in the registry""" + return self._obbjects diff --git a/cli/openbb_cli/assets/i18n/en.yml b/cli/openbb_cli/assets/i18n/en.yml index a8d06e45b884..05a772b73204 100644 --- a/cli/openbb_cli/assets/i18n/en.yml +++ b/cli/openbb_cli/assets/i18n/en.yml @@ -34,3 +34,5 @@ en: settings/language: translation language settings/n_rows: number of rows to show on non interactive tables settings/n_cols: number of columns to show on non interactive tables + settings/obbject_msg: show obbject registry message after a new result is added + settings/obbject_res: define the maximum number of obbjects allowed in the registry diff --git a/cli/openbb_cli/controllers/base_controller.py b/cli/openbb_cli/controllers/base_controller.py index a3d8b20aa162..987f0cd413b5 100644 --- a/cli/openbb_cli/controllers/base_controller.py +++ b/cli/openbb_cli/controllers/base_controller.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Union +import pandas as pd from openbb_cli.config import setup from openbb_cli.config.completer import NestedCompleter from openbb_cli.config.constants import SCRIPT_TAGS @@ -20,6 +21,7 @@ get_flair_and_username, parse_and_split_input, print_guest_block_msg, + print_rich_table, remove_file, system_clear, ) @@ -64,6 +66,7 @@ class BaseController(metaclass=ABCMeta): "stop", "hold", "whoami", + "results", ] CHOICES_COMMANDS: List[str] = [] @@ -119,6 +122,11 @@ def __init__(self, queue: Optional[List[str]] = None) -> None: self.parser.exit_on_error = False # type: ignore self.parser.add_argument("cmd", choices=self.controller_choices) + def update_completer(self, choices) -> None: + """Update the completer with new choices.""" + if session.prompt_session and session.settings.USE_PROMPT_TOOLKIT: + self.completer = NestedCompleter.from_nested_dict(choices) + def check_path(self) -> None: """Check if command path is valid.""" path = self.PATH @@ -732,6 +740,30 @@ def call_whoami(self, other_args: List[str]) -> None: else: print_guest_block_msg() + def call_results(self, other_args: List[str]): + """Process results command.""" + parser = argparse.ArgumentParser( + add_help=False, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + prog="results", + description="Process results command. This command displays a registry of " + "'OBBjects' where all execution results are stored. " + "It is organized as a stack, with the most recent result at index 0.", + ) + ns_parser = self.parse_simple_args(parser, other_args) + if ns_parser: + results = session.obbject_registry.all + if results: + df = pd.DataFrame.from_dict(results, orient="index") + print_rich_table( + df, + show_index=True, + index_name="stack index", + title="OBBject Results", + ) + else: + session.console.print("[info]No results found.[/info]") + @staticmethod def parse_simple_args(parser: argparse.ArgumentParser, other_args: List[str]): """Parse list of arguments into the supplied parser. diff --git a/cli/openbb_cli/controllers/base_platform_controller.py b/cli/openbb_cli/controllers/base_platform_controller.py index 0fc55c0ff1ca..6f236f27f951 100644 --- a/cli/openbb_cli/controllers/base_platform_controller.py +++ b/cli/openbb_cli/controllers/base_platform_controller.py @@ -11,8 +11,6 @@ from openbb_cli.argparse_translator.argparse_class_processor import ( ArgparseClassProcessor, ) -from openbb_cli.argparse_translator.obbject_registry import Registry -from openbb_cli.config.completer import NestedCompleter from openbb_cli.config.menu_text import MenuText from openbb_cli.controllers.base_controller import BaseController from openbb_cli.controllers.utils import export_data, print_rich_table @@ -69,26 +67,23 @@ def __init__( self._link_obbject_to_data_processing_commands() self._generate_commands() self._generate_sub_controllers() - - if session.prompt_session and session.settings.USE_PROMPT_TOOLKIT: - choices: dict = self.choices_default - self.completer = NestedCompleter.from_nested_dict(choices) + self.update_completer(self.choices_default) def _link_obbject_to_data_processing_commands(self): """Link data processing commands to OBBject registry.""" for _, trl in self.translators.items(): for action in trl._parser._actions: # pylint: disable=protected-access if action.dest == "data": - action.choices = range(len(Registry.obbjects)) + action.choices = range(len(session.obbject_registry.obbjects)) action.type = int action.nargs = None def _intersect_data_processing_commands(self, ns_parser): """Intersect data processing commands and change the obbject id into an actual obbject.""" if hasattr(ns_parser, "data") and ns_parser.data in range( - len(Registry.obbjects) + len(session.obbject_registry.obbjects) ): - obbject = Registry.get(ns_parser.data) + obbject = session.obbject_registry.get(ns_parser.data) setattr(ns_parser, "data", obbject.results) return ns_parser @@ -159,7 +154,22 @@ def method(self, other_args: List[str], translator=translator): title = f"{self.PATH}{translator.func.__name__}" if obbject: - Registry.register(obbject) + max_obbjects_exceeded = ( + len(session.obbject_registry.obbjects) + >= session.settings.N_TO_KEEP_OBBJECT_REGISTRY + ) + if max_obbjects_exceeded: + session.obbject_registry.remove() + + session.obbject_registry.register(obbject) + # we need to force to re-link so that the new obbject + # is immediately available for data processing commands + self._link_obbject_to_data_processing_commands() + # also update the completer + self.update_completer(self.choices_default) + + if session.settings.SHOW_MSG_OBBJECT_REGISTRY: + session.console.print("Added OBBject to registry.") if hasattr(ns_parser, "chart") and ns_parser.chart: obbject.show() @@ -195,6 +205,11 @@ def method(self, other_args: List[str], translator=translator): figure=fig, ) + if max_obbjects_exceeded: + session.console.print( + "[yellow]\nMaximum number of OBBjects reached. The oldest entry was removed.[yellow]" + ) + except Exception as e: session.console.print(f"[red]{e}[/]\n") return diff --git a/cli/openbb_cli/controllers/cli_controller.py b/cli/openbb_cli/controllers/cli_controller.py index e82a5dbdfc0b..dfc5eb878dd6 100644 --- a/cli/openbb_cli/controllers/cli_controller.py +++ b/cli/openbb_cli/controllers/cli_controller.py @@ -20,9 +20,7 @@ import pandas as pd import requests from openbb import obb -from openbb_cli.argparse_translator.obbject_registry import Registry from openbb_cli.config import constants -from openbb_cli.config.completer import NestedCompleter from openbb_cli.config.constants import ( ASSETS_DIRECTORY, ENV_FILE_SETTINGS, @@ -217,7 +215,7 @@ def update_runtime_choices(self): "--tag3": {c: None for c in constants.SCRIPT_TAGS}, } - self.completer = NestedCompleter.from_nested_dict(choices) + self.update_completer(choices) def print_help(self): """Print help.""" @@ -303,11 +301,11 @@ def parse_input(self, an_input: str) -> List: def call_settings(self, _): """Process feature flags command.""" - from openbb_cli.controllers.feature_flags_controller import ( - FeatureFlagsController, + from openbb_cli.controllers.settings_controller import ( + SettingsController, ) - self.queue = self.load_class(FeatureFlagsController, self.queue) + self.queue = self.load_class(SettingsController, self.queue) def call_exe(self, other_args: List[str]): """Process exe command.""" @@ -475,17 +473,6 @@ def call_exe(self, other_args: List[str]): ) self.queue = self.queue[1:] - def call_results(self, _): - """Process results command.""" - results = Registry().all - if results: - df = pd.DataFrame.from_dict(results, orient="index") - print_rich_table( - df, show_index=True, index_name="stack index", title="OBBject Results" - ) - else: - session.console.print("[info]No results found.[/info]") - def handle_job_cmds(jobs_cmds: Optional[List[str]]) -> Optional[List[str]]: """Handle job commands.""" diff --git a/cli/openbb_cli/controllers/feature_flags_controller.py b/cli/openbb_cli/controllers/settings_controller.py similarity index 86% rename from cli/openbb_cli/controllers/feature_flags_controller.py rename to cli/openbb_cli/controllers/settings_controller.py index d6de826964bd..d293078a0efd 100644 --- a/cli/openbb_cli/controllers/feature_flags_controller.py +++ b/cli/openbb_cli/controllers/settings_controller.py @@ -3,7 +3,6 @@ import argparse from typing import List, Optional -from openbb_cli.config.completer import NestedCompleter from openbb_cli.config.constants import AVAILABLE_FLAIRS from openbb_cli.config.menu_text import MenuText @@ -16,7 +15,7 @@ session = Session() -class FeatureFlagsController(BaseController): +class SettingsController(BaseController): """Feature Flags Controller class.""" CHOICES_COMMANDS: List[str] = [ @@ -40,6 +39,8 @@ class FeatureFlagsController(BaseController): "language", "n_rows", "n_cols", + "obbject_msg", + "obbject_res", ] PATH = "/settings/" CHOICES_GENERATION = True @@ -48,9 +49,7 @@ def __init__(self, queue: Optional[List[str]] = None): """Initialize the Constructor.""" super().__init__(queue) - if session.prompt_session and session.settings.USE_PROMPT_TOOLKIT: - choices: dict = self.choices_default - self.completer = NestedCompleter.from_nested_dict(choices) + self.update_completer(self.choices_default) def print_help(self): """Print help.""" @@ -68,6 +67,7 @@ def print_help(self): mt.add_setting("tbhint", settings.TOOLBAR_HINT) mt.add_setting("overwrite", settings.FILE_OVERWRITE) mt.add_setting("version", settings.SHOW_VERSION) + mt.add_setting("obbject_msg", settings.SHOW_MSG_OBBJECT_REGISTRY) mt.add_raw("\n") mt.add_info("_settings_") mt.add_raw("\n") @@ -77,6 +77,7 @@ def print_help(self): mt.add_cmd("language") mt.add_cmd("n_rows") mt.add_cmd("n_cols") + mt.add_cmd("obbject_res") session.console.print(text=mt.menu_text, menu="Feature Flags") @@ -134,6 +135,13 @@ def call_tbhint(self, _): session.console.print("Will take effect when running CLI again.") session.settings.set_item("TOOLBAR_HINT", not session.settings.TOOLBAR_HINT) + def call_obbject_msg(self, _): + """Process obbject_msg command.""" + session.settings.set_item( + "SHOW_MSG_OBBJECT_REGISTRY", + not session.settings.SHOW_MSG_OBBJECT_REGISTRY, + ) + def call_console_style(self, other_args: List[str]) -> None: """Process cosole_style command.""" parser = argparse.ArgumentParser( @@ -290,3 +298,30 @@ def call_n_cols(self, other_args: List[str]) -> None: session.console.print( f"Current number of columns: {session.settings.ALLOWED_NUMBER_OF_COLUMNS}" ) + + def call_obbject_res(self, other_args: List[str]): + """Process obbject_res command.""" + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + prog="obbject_res", + description="Maximum allowed number of results to keep in the OBBject Registry.", + add_help=False, + ) + parser.add_argument( + "-n", + "--number", + dest="number", + action="store", + required=False, + type=int, + ) + ns_parser = self.parse_simple_args(parser, other_args) + + if ns_parser and ns_parser.number: + session.settings.set_item("N_TO_KEEP_OBBJECT_REGISTRY", ns_parser.number) + + elif not other_args: + session.console.print( + f"Current maximum allowed number of results to keep in the OBBject registry:" + f" {session.settings.N_TO_KEEP_OBBJECT_REGISTRY}" + ) diff --git a/cli/openbb_cli/models/settings.py b/cli/openbb_cli/models/settings.py index 754b122034cc..b4380eee1c06 100644 --- a/cli/openbb_cli/models/settings.py +++ b/cli/openbb_cli/models/settings.py @@ -29,19 +29,21 @@ class Settings(BaseModel): REMEMBER_CONTEXTS: bool = True ENABLE_RICH_PANEL: bool = True TOOLBAR_HINT: bool = True + SHOW_MSG_OBBJECT_REGISTRY: bool = False # GENERAL TIMEZONE: str = "America/New_York" FLAIR: str = ":openbb" USE_LANGUAGE: str = "en" PREVIOUS_USE: bool = False + N_TO_KEEP_OBBJECT_REGISTRY: int = 10 # STYLE RICH_STYLE: str = "dark" # OUTPUT - ALLOWED_NUMBER_OF_ROWS: int = 366 - ALLOWED_NUMBER_OF_COLUMNS: int = 15 + ALLOWED_NUMBER_OF_ROWS: int = 20 + ALLOWED_NUMBER_OF_COLUMNS: int = 5 # OPENBB HUB_URL: str = "https://my.openbb.co" diff --git a/cli/openbb_cli/session.py b/cli/openbb_cli/session.py index ef63fd6e7f03..be2a7938aa31 100644 --- a/cli/openbb_cli/session.py +++ b/cli/openbb_cli/session.py @@ -9,6 +9,7 @@ from openbb_core.app.model.user_settings import UserSettings as User from prompt_toolkit import PromptSession +from openbb_cli.argparse_translator.obbject_registry import Registry from openbb_cli.config.completer import CustomFileHistory from openbb_cli.config.console import Console from openbb_cli.config.constants import HIST_FILE_PROMPT @@ -31,6 +32,7 @@ def __init__(self): settings=self._settings, style=self._style.console_style ) self._prompt_session = self._get_prompt_session() + self._obbject_registry = Registry() @property def user(self) -> User: @@ -52,6 +54,11 @@ def console(self) -> Console: """Get console.""" return self._console + @property + def obbject_registry(self) -> Registry: + """Get obbject registry.""" + return self._obbject_registry + @property def prompt_session(self) -> Optional[PromptSession]: """Get prompt session."""