diff --git a/README.md b/README.md index 90f5e08..f857517 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,33 @@ -# tap-pubg +# `tap-pubg` -`tap-pubg` is a Singer tap for pubg. +pubg tap class. -Built with the [Meltano Tap SDK](https://sdk.meltano.com) for Singer Taps. +Built with the [Meltano SDK](https://sdk.meltano.com) for Singer Taps and Targets. -## Installation +## Capabilities + +* `catalog` +* `state` +* `discover` +* `about` +* `stream-maps` +* `schema-flattening` + +## Settings + +| Setting | Required | Default | Description | +|:--------------------|:--------:|:-------:|:------------| +| api_key | True | None | The token to authenticate against the API service | +| player_names | True | None | Player Names sepearted by commas that you'd like to pull, maximum of 10 | +| platform | True | None | platform, ie steam | +| stream_maps | False | None | Config object for stream maps capability. | +| stream_map_config | False | None | User-defined config values to be used within map expressions. | +| flattening_enabled | False | None | 'True' to enable schema flattening and automatically expand nested properties. | +| flattening_max_depth| False | None | The max depth to flatten schemas. | + +A full list of supported settings and capabilities is available by running: `tap-pubg --about` -- [ ] `Developer TODO:` Update the below as needed to correctly describe the install procedure. For instance, if you do not have a PyPi repo, or if you want users to directly install from your git repo, you can modify this step as appropriate. -```bash -pipx install tap-pubg -``` ## Configuration diff --git a/tap_pubg/client.py b/tap_pubg/client.py index 3a76cad..08306e5 100644 --- a/tap_pubg/client.py +++ b/tap_pubg/client.py @@ -16,7 +16,7 @@ class PubgStream(RESTStream): """pubg stream class.""" - + _LOG_REQUEST_METRIC_URLS=True @property def url_base(self) -> str: """Return the API URL root, configurable via tap settings.""" @@ -71,9 +71,6 @@ def get_url_params( params: dict = {} if next_page_token: params["page"] = next_page_token - if self.replication_key: - params["sort"] = "asc" - params["order_by"] = self.replication_key return params def parse_response(self, response: requests.Response) -> Iterable[dict]: diff --git a/tap_pubg/schemas/match.json b/tap_pubg/schemas/match.json index 48082bf..0af1436 100644 --- a/tap_pubg/schemas/match.json +++ b/tap_pubg/schemas/match.json @@ -1,11 +1,278 @@ { "type": "object", "properties": { - "type": { + "createdAt": { + "type": "string", + "format": "date-time" + }, + "URL": { "type": "string" }, - "id": { - "type": "string" + "data": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "attributes": { + "type": "object", + "properties": { + "shardId": { + "type": "string" + }, + "tags": { + "type": "null" + }, + "mapName": { + "type": "string" + }, + "isCustomMatch": { + "type": "boolean" + }, + "seasonState": { + "type": "string" + }, + "gameMode": { + "type": "string" + }, + "duration": { + "type": "integer" + }, + "stats": { + "type": "null" + }, + "titleId": { + "type": "string" + }, + "matchType": { + "type": "string" + }, + "createdAt": { + "type": "string" + } + } + }, + "relationships": { + "type": "object", + "properties": { + "rosters": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "id": { + "type": "string" + } + } + } + } + } + }, + "assets": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "id": { + "type": "string" + } + } + } + } + } + } + } + }, + "links": { + "type": "object", + "properties": { + "self": { + "type": "string" + }, + "schema": { + "type": "string" + } + } + } + } + }, + "included": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "attributes": { + "type": "object", + "properties": { + "stats": { + "type": "object", + "properties": { + "DBNOs": { + "type": "integer" + }, + "assists": { + "type": "integer" + }, + "boosts": { + "type": "integer" + }, + "damageDealt": { + "type": "number" + }, + "deathType": { + "type": "string" + }, + "headshotKills": { + "type": "integer" + }, + "heals": { + "type": "integer" + }, + "killPlace": { + "type": "integer" + }, + "killStreaks": { + "type": "integer" + }, + "kills": { + "type": "integer" + }, + "longestKill": { + "type": "number" + }, + "name": { + "type": "string" + }, + "playerId": { + "type": "string" + }, + "revives": { + "type": "integer" + }, + "rideDistance": { + "type": "number" + }, + "roadKills": { + "type": "integer" + }, + "swimDistance": { + "type": "number" + }, + "teamKills": { + "type": "integer" + }, + "timeSurvived": { + "type": "number" + }, + "vehicleDestroys": { + "type": "integer" + }, + "walkDistance": { + "type": "number" + }, + "weaponsAcquired": { + "type": "integer" + }, + "winPlace": { + "type": "integer" + }, + "rank": { + "type": "integer" + }, + "teamId": { + "type": "integer" + } + } + }, + "actor": { + "type": "string" + }, + "shardId": { + "type": "string" + }, + "won": { + "type": "string" + }, + "description": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "URL": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "relationships": { + "type": "object", + "properties": { + "participants": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "id": { + "type": "string" + } + } + } + } + } + }, + "team": { + "type": "object", + "properties": { + "data": { + "type": "null" + } + } + } + } + } + } + } + }, + "links": { + "type": "object", + "properties": { + "self": { + "type": "string" + } + } + }, + "meta": { + "type": "object" } } } diff --git a/tap_pubg/schemas/matches.json b/tap_pubg/schemas/matches.json deleted file mode 100644 index 8392705..0000000 --- a/tap_pubg/schemas/matches.json +++ /dev/null @@ -1,271 +0,0 @@ -{ - "type": "object", - "properties": { - "data": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "attributes": { - "type": "object", - "properties": { - "shardId": { - "type": "string" - }, - "tags": { - "type": "null" - }, - "mapName": { - "type": "string" - }, - "isCustomMatch": { - "type": "boolean" - }, - "seasonState": { - "type": "string" - }, - "gameMode": { - "type": "string" - }, - "duration": { - "type": "integer" - }, - "stats": { - "type": "null" - }, - "titleId": { - "type": "string" - }, - "matchType": { - "type": "string" - }, - "createdAt": { - "type": "string" - } - } - }, - "relationships": { - "type": "object", - "properties": { - "rosters": { - "type": "object", - "properties": { - "data": { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - } - } - } - } - } - }, - "assets": { - "type": "object", - "properties": { - "data": { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - } - } - } - } - } - } - } - }, - "links": { - "type": "object", - "properties": { - "self": { - "type": "string" - }, - "schema": { - "type": "string" - } - } - } - } - }, - "included": { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "attributes": { - "type": "object", - "properties": { - "stats": { - "type": "object", - "properties": { - "DBNOs": { - "type": "integer" - }, - "assists": { - "type": "integer" - }, - "boosts": { - "type": "integer" - }, - "damageDealt": { - "type": "number" - }, - "deathType": { - "type": "string" - }, - "headshotKills": { - "type": "integer" - }, - "heals": { - "type": "integer" - }, - "killPlace": { - "type": "integer" - }, - "killStreaks": { - "type": "integer" - }, - "kills": { - "type": "integer" - }, - "longestKill": { - "type": "number" - }, - "name": { - "type": "string" - }, - "playerId": { - "type": "string" - }, - "revives": { - "type": "integer" - }, - "rideDistance": { - "type": "number" - }, - "roadKills": { - "type": "integer" - }, - "swimDistance": { - "type": "number" - }, - "teamKills": { - "type": "integer" - }, - "timeSurvived": { - "type": "number" - }, - "vehicleDestroys": { - "type": "integer" - }, - "walkDistance": { - "type": "number" - }, - "weaponsAcquired": { - "type": "integer" - }, - "winPlace": { - "type": "integer" - }, - "rank": { - "type": "integer" - }, - "teamId": { - "type": "integer" - } - } - }, - "actor": { - "type": "string" - }, - "shardId": { - "type": "string" - }, - "won": { - "type": "string" - }, - "description": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "URL": { - "type": "string" - }, - "name": { - "type": "string" - } - } - }, - "relationships": { - "type": "object", - "properties": { - "participants": { - "type": "object", - "properties": { - "data": { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - } - } - } - } - } - }, - "team": { - "type": "object", - "properties": { - "data": { - "type": "null" - } - } - } - } - } - } - } - }, - "links": { - "type": "object", - "properties": { - "self": { - "type": "string" - } - } - }, - "meta": { - "type": "object" - } - } -} diff --git a/tap_pubg/schemas/player_matches.json b/tap_pubg/schemas/player_matches.json new file mode 100644 index 0000000..6b84d79 --- /dev/null +++ b/tap_pubg/schemas/player_matches.json @@ -0,0 +1,8 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + } + } +} diff --git a/tap_pubg/schemas/telemetry.json b/tap_pubg/schemas/telemetry.json index 3062744..294b27a 100644 --- a/tap_pubg/schemas/telemetry.json +++ b/tap_pubg/schemas/telemetry.json @@ -4,6 +4,10 @@ "URL": { "type": "string" }, + "createdAt": { + "type": "string", + "format": "date-time" + }, "data":{ "type": "array", "items": { diff --git a/tap_pubg/streams.py b/tap_pubg/streams.py index c3fd2ba..0e67c03 100644 --- a/tap_pubg/streams.py +++ b/tap_pubg/streams.py @@ -10,10 +10,7 @@ from singer_sdk.plugin_base import PluginBase as TapBaseClass from tap_pubg.client import PubgStream -# TODO: Delete this is if not using json files for schema definition SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") -# TODO: - Override `UsersStream` and `GroupsStream` with your own stream definition. -# - Copy-paste as many times as needed to create multiple stream types. class PlayersStream(PubgStream): """Define custom stream.""" @@ -29,10 +26,11 @@ def path(self): class PlayerMatches(PubgStream): - name = "player_matches" + """Need seperate records for each match a player has. Note that this is only the last 14 days of Matches""" + name = "player_match" primary_keys = ["id"] replication_key = None - schema_filepath = SCHEMAS_DIR / "match.json" + schema_filepath = SCHEMAS_DIR / "player_matches.json" records_jsonpath = "$.data[0].relationships.matches.data[*]" @property @@ -46,44 +44,56 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: "match_id": record["id"], } - - class MatchesStream(PubgStream): "Matches data" - name = "matches" + name = "match" primary_keys = ["id"] - replication_key = None - schema_filepath = SCHEMAS_DIR / "matches.json" + replication_key = "createdAt" + schema_filepath = SCHEMAS_DIR / "match.json" path = "/matches/{match_id}" parent_stream_type = PlayerMatches + def post_process(self, row: dict, context: Optional[dict]) -> dict: + """As needed, append or transform raw data to match expected structure.""" + included = row["included"] #List of Dicts + teletermy_includes = list(filter(lambda included: included["type"]=="asset" and included["attributes"]["name"]=="telemetry", included)) + if (len(teletermy_includes) != 1): + self.logger.warning(f"Record did not contain telemetry info, skipping. Record data: {json.dumps(record)}") + #TODO test to see if running this while a match is going will fail due to there not being a telemetry record + return None + telemetry_data = { + "URL": teletermy_includes[0]["attributes"]["URL"], + #Used this isntead of match attributes as it's possible we query a match while it's happening. If we don't have telemetry data we don't want to mark this as complete + "createdAt": teletermy_includes[0]["attributes"]["createdAt"], + } + row.update(telemetry_data) + return row + #What if we want other child streams here beyond telemtry? #I think we'd just expand this to include them, and all the relevent child streams would use only the data they needed - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: """Return a context dictionary for child streams.""" - included = record["included"] #List of Dicts - teletermy_includes = list(filter(lambda included: included["type"]=="asset" and included["attributes"]["name"]=="telemetry", included)) - if (len(teletermy_includes) != 1): - self.logger.error(f"Record JSON data for failure: {json.dumps(record)}") - raise Exception("Matches includes more than one telemtry record, I'm dead") + self.logger.info(f"record: {record}") return { - "URL": teletermy_includes[0]["attributes"]["URL"], + "URL": record["URL"], } + class TelemetryStream(PubgStream): "Telemetry Data" name = "match_telemetry" - primary_keys = ["MatchId"] + primary_keys = ["URL"] + #TODO if parent stream is called will this always be called? replication_key = None schema_filepath = SCHEMAS_DIR / "telemetry.json" path = "" - parent_stream_type = MatchesStream + def get_url(self, context: Optional[dict]): """Return the API URL root, configurable via tap settings.""" + self.logger.info(f"context in get_url: {context}") return context["URL"] def parse_response(self, response: requests.Response) -> Iterable[dict]: diff --git a/tap_pubg/tap.py b/tap_pubg/tap.py index 0156207..e23274b 100644 --- a/tap_pubg/tap.py +++ b/tap_pubg/tap.py @@ -9,13 +9,13 @@ PlayersStream, PlayerMatches, MatchesStream, - TelemetryStream, +# TelemetryStream, ) STREAM_TYPES = [ PlayersStream, PlayerMatches, - TelemetryStream, MatchesStream, +# TelemetryStream, ] @@ -34,7 +34,7 @@ class Tappubg(Tap): "player_names", th.ArrayType(th.StringType), required=True, - description="Player Names you'd like to pull" + description="Player Names sepearted by commas that you'd like to pull, maximum of 10" ), th.Property( "platform",