diff --git a/README.md b/README.md index c50fc70..19adecb 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Singer Tap for Jotform. Built with the [Meltano Singer SDK](https://sdk.meltano. | api_url | False | https://api.jotform.com | API Base URL | | user_agent | False | tap-jotform/0.0.1 | User-Agent header | | start_date | False | None | Start date for data collection | +| requests_cache_config| False | None | Cache configuration for HTTP requests | | stream_maps | False | None | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). | | stream_map_config | False | None | User-defined config values to be used within map expressions. | | flattening_enabled | False | None | 'True' to enable schema flattening and automatically expand nested properties. | diff --git a/meltano.yml b/meltano.yml index 3f6924c..43d2abe 100644 --- a/meltano.yml +++ b/meltano.yml @@ -35,6 +35,17 @@ plugins: kind: date_iso8601 label: Start Date description: Start date for collecting data + - name: requests_cache.enabled + kind: boolean + label: Requests Cache Enabled + description: Enable requests cache + - name: requests_cache.config.expire_after + kind: integer + label: Requests Cache Expire After + description: Requests cache expire after + config: + requests_cache.enabled: true + requests_cache.config.expire_after: 3600 loaders: - name: target-duckdb variant: jwills diff --git a/tap_jotform/client.py b/tap_jotform/client.py index 5adb3dd..0cff99d 100644 --- a/tap_jotform/client.py +++ b/tap_jotform/client.py @@ -10,8 +10,6 @@ from singer_sdk.pagination import BaseOffsetPaginator from singer_sdk.streams import RESTStream -requests_cache.install_cache("requests_cache") - class JotformPaginator(BaseOffsetPaginator): """Jotform pagination class.""" @@ -111,6 +109,22 @@ def parse_response( ) yield from super().parse_response(response) + @property + def requests_session(self) -> requests.Session: + """Return a new requests session object. + + Returns: + A new requests session object. + """ + if ( + self.config.get("requests_cache") + and self.config["requests_cache"]["enabled"] + ): + self._requests_session = requests_cache.CachedSession( + **self.config["requests_cache"]["config"] + ) + return super().requests_session + class JotformPaginatedStream(JotformStream): """A Jotform stream with pagination.""" diff --git a/tap_jotform/tap.py b/tap_jotform/tap.py index bb17cb8..448701f 100644 --- a/tap_jotform/tap.py +++ b/tap_jotform/tap.py @@ -58,6 +58,30 @@ def config_jsonschema(cls) -> dict: # type: ignore required=False, description="Start date for data collection", ), + th.Property( + "requests_cache", + th.ObjectType( + th.Property( + "enabled", + th.BooleanType, + default=False, + description="Enable requests cache", + ), + th.Property( + "config", + th.ObjectType( + th.Property( + "expire_after", + th.IntegerType, + description="Cache expiration time in seconds", + ), + ), + description="Requests cache configuration", + default={}, + ), + ), + description="Cache configuration for HTTP requests", + ), ).to_dict() def discover_streams(self) -> list[Stream]: