From d5912f9daae102291934f09ca5044e8ecb22568d Mon Sep 17 00:00:00 2001 From: Damian Parrino Date: Wed, 30 Aug 2023 10:38:45 -0300 Subject: [PATCH] fixes --- docs/4.tools/events.md | 67 ------------------------------------ docs/4.tools/near-lake.md | 71 +++++++++++++++++++++++++++++++++++++++ website/sidebars.json | 9 ++--- 3 files changed, 76 insertions(+), 71 deletions(-) create mode 100644 docs/4.tools/near-lake.md diff --git a/docs/4.tools/events.md b/docs/4.tools/events.md index e05906c3d37..7d471847040 100644 --- a/docs/4.tools/events.md +++ b/docs/4.tools/events.md @@ -69,70 +69,3 @@ that listens for **all** `nft_mint` and `nft_transfer` events in the NEAR networ ::: --- - -## NEAR Lake Indexer - -NEAR Lake is an indexer built on top of [NEAR Indexer Framework](https://near-indexers.io/docs/projects/near-indexer-framework) to watch the network and store all the events as JSON files on AWS S3. - -:::info GitHub repo - -You can find the Lake Indexer source code in [this GitHub repository](https://github.com/near/near-lake-indexer/). - -::: - -### How it works - -:::tip - -[Pagoda Inc.](https://pagoda.co) runs NEAR Lake nodes to store the data in JSON format on AWS S3. -There is no need to run your own NEAR Lake unless you have specific reasons to do that. - -::: - -There are AWS S3 buckets created: - -- `near-lake-data-testnet` (`eu-central-1` region) -- `near-lake-data-mainnet` (`eu-central-1` region) - -All the buckets are set up the way the requester pays for the access. Anyone can read from these buckets by connecting to them with their own AWS credentials to be charged by Amazon. - -### Data structure - -The data structure used by Lake Indexer is the following: - -``` - / - block.json - shard_0.json - shard_1.json - ... - shard_N.json -``` - -`` is a 12-character-long [`u64`](https://doc.rust-lang.org/std/primitive.u64.html) string with leading zeros (e.g "000042839521"). See [this issue for reasoning](https://github.com/near/near-lake/issues/23). - -`block_json` contains JSON-serialized `BlockView` struct. **NB!** this struct might change in the future, we will announce it - -`shard_N.json` where N is [`u64`](https://doc.rust-lang.org/std/primitive.u64.html) starting from `0`. Represents the index number of the shard. In order to find out the expected number of shards in the block you can look in `block.json` at `.header.chunks_included` - - -### How to use it - -We have created the [NEAR Lake Framework](https://near-indexers.io/docs/projects/near-lake-framework) to have a simple straightforward way to create an indexer on top of the data stored by NEAR Lake itself. - -:::info NEAR Lake Framework - -You can check the NEAR Lake Framework release announcement on the [NEAR Governance Forum](https://gov.near.org/t/announcement-near-lake-framework-brand-new-word-in-indexer-building-approach/17668). - -::: - -We have prepared this video tutorial with a simple example to give you an overview and some practical ideas. - - diff --git a/docs/4.tools/near-lake.md b/docs/4.tools/near-lake.md new file mode 100644 index 00000000000..9e2891f7c62 --- /dev/null +++ b/docs/4.tools/near-lake.md @@ -0,0 +1,71 @@ +--- +id: near-lake +sidebar_label: Lake Overview +title: NEAR Lake Indexer +--- + + +NEAR Lake is an indexer built on top of [NEAR Indexer Framework](https://near-indexers.io/docs/projects/near-indexer-framework) to watch the network and store all the [events](events.md) as JSON files on AWS S3. + +:::info GitHub repo + +You can find the Lake Indexer source code in [this GitHub repository](https://github.com/near/near-lake-indexer/). + +::: + +### How it works + +:::tip + +[Pagoda Inc.](https://pagoda.co) runs NEAR Lake nodes to store the data in JSON format on AWS S3. +There is no need to run your own NEAR Lake unless you have specific reasons to do that. + +::: + +There are AWS S3 buckets created: + +- `near-lake-data-testnet` (`eu-central-1` region) +- `near-lake-data-mainnet` (`eu-central-1` region) + +All the buckets are set up the way the requester pays for the access. Anyone can read from these buckets by connecting to them with their own AWS credentials to be charged by Amazon. + +### Data structure + +The data structure used by Lake Indexer is the following: + +``` + / + block.json + shard_0.json + shard_1.json + ... + shard_N.json +``` + +`` is a 12-character-long [`u64`](https://doc.rust-lang.org/std/primitive.u64.html) string with leading zeros (e.g "000042839521"). See [this issue for reasoning](https://github.com/near/near-lake/issues/23). + +`block_json` contains JSON-serialized `BlockView` struct. **NB!** this struct might change in the future, we will announce it + +`shard_N.json` where N is [`u64`](https://doc.rust-lang.org/std/primitive.u64.html) starting from `0`. Represents the index number of the shard. In order to find out the expected number of shards in the block you can look in `block.json` at `.header.chunks_included` + + +### How to use it + +We have created the [NEAR Lake Framework](/concepts/advanced/near-lake-framework) to have a simple straightforward way to create an indexer on top of the data stored by NEAR Lake itself. + +:::info NEAR Lake Framework + +You can check the NEAR Lake Framework release announcement on the [NEAR Governance Forum](https://gov.near.org/t/announcement-near-lake-framework-brand-new-word-in-indexer-building-approach/17668). + +::: + +We have prepared this video tutorial with a simple example to give you an overview and some practical ideas. + + diff --git a/website/sidebars.json b/website/sidebars.json index 526b81857b6..61b70209940 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -214,9 +214,9 @@ }, { "NEAR Lake Framework": [ - "tutorials/indexer/migrating-to-near-lake-framework", - "tools/realtime", + "tools/near-lake", "tutorials/indexer/near-lake-state-changes-indexer", + "tutorials/indexer/migrating-to-near-lake-framework", { "Building Indexers": [ "tutorials/indexer/js-lake-indexer", "tutorials/indexer/python-lake-indexer", @@ -649,8 +649,9 @@ ] }, { - "QueryAPI Indexing": ["bos/dev/intro" - ] + "type": "link", + "label": "QueryAPI Indexing ↗", + "href": "/bos/queryapi/intro" }, { "type": "html",