Change Uniswap v3 subgraph queries to use pagination

.github/workflows/test-on-demand.yml

@@ -0,0 +1,60 @@
+name: Test_Change_On_Demand
+on:
+  workflow_dispatch:
+    secrets:
+      ALCHEMY_CONNECTION_ARBITRUM:
+        required: true
+      ALCHEMY_CONNECTION_ETHEREUM:
+        required: true
+      ALCHEMY_CONNECTION_POLYGON:
+        required: true
+      GRAPH_API_KEY:
+        required: true
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - id: file_changes
+        uses: trilom/[email protected]
+        with:
+          output: 'json'
+          fileOutput: 'json'
+          pushBefore: 279f1aac290c38c5aeef00d7484ae3fb8e8d7929
+          pushAfter: 3a131965cb5d0463c0d2f60c3a9e8fb339b798ab
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Run changes files through test script
+        env:
+          ALCHEMY_CONNECTION_ARBITRUM: ${{ secrets.ALCHEMY_CONNECTION_ARBITRUM }}
+          ALCHEMY_CONNECTION_ETHEREUM: ${{ secrets.ALCHEMY_CONNECTION_ETHEREUM }}
+          ALCHEMY_CONNECTION_POLYGON: ${{ secrets.ALCHEMY_CONNECTION_POLYGON }}
+          GRAPH_API_KEY: ${{ secrets.GRAPH_API_KEY }}
+        run: |
+          RUN_FILES=$(
+            MODIFIED=${{ steps.file_changes.outputs.files_modified}} \
+            ADDED=${{ steps.file_changes.outputs.files_added}} \
+            node ${{ github.workspace }}/.github/workflows/getFileList.js
+          )
+
+          if [ "$RUN_FILES" = "[]" ]; then
+            echo "No adapter files were modified"
+            exit 0
+          fi
+
+          npm ci
+
+          for i in $(echo $RUN_FILES | tr  -d '"[]' | tr "," "\n")
+          do
+            {
+              npm run test --adapter=${i} 2>&1 | tee output.txt
+              node ${{ github.workspace }}/.github/workflows/commentResult.js /home/runner/work/yield-server/yield-server/output.txt "${{ github.repository_owner }}" "${{ github.event.repository.name }}" "${{ github.event.number }}" ${i}
+              if grep -q "PASS src/adaptors/test.js" output.txt; then
+                exit 0;
+              else
+                exit 1;
+              fi
+            } || {
+              echo -n $i
+              echo ' doesnt run'
+            }
+          done

src/adaptors/uniswap-v3/index.js

@@ -34,7 +34,7 @@ const chains = {
 
 const query = gql`
   {
-    pools(first: 1000, orderBy: totalValueLockedUSD, orderDirection: desc block: {number: <PLACEHOLDER>}) {
+    pools(first: 1000, skip: <SKIP>, where: {totalValueLockedUSD_gt: "10000"}, orderBy: totalValueLockedUSD, orderDirection: desc block: {number: <PLACEHOLDER>}) {
       id
       totalValueLockedToken0
       totalValueLockedToken1
@@ -56,9 +56,9 @@ const query = gql`
 
 const queryPrior = gql`
   {
-    pools( first: 1000 orderBy: totalValueLockedUSD orderDirection:desc block: {number: <PLACEHOLDER>}) {
-      id 
-      volumeUSD 
+    pools(first: 1000, skip: <SKIP>, where: {totalValueLockedUSD_gt: "10000"}, orderBy: totalValueLockedUSD orderDirection:desc block: {number: <PLACEHOLDER>}) {
+      id
+      volumeUSD
     }
   }
 `;
@@ -85,9 +85,17 @@ const topLvl = async (
     );
 
     // pull data
-    let queryC = query;
-    let dataNow = await request(url, queryC.replace('<PLACEHOLDER>', block));
-    dataNow = dataNow.pools;
+    let allPools = [];
+    let skip = 0;
+    while (true) {
+      let queryC = query;
+      console.log(`allPools: query pools from: ${skip} to: ${skip + 1000}`);
+      let currentData = await request(url, queryC.replace('<PLACEHOLDER>', block).replace('<SKIP>', skip));
+      if (!currentData.pools || currentData.pools.length === 0) break;
+      allPools = [...allPools, ...currentData.pools];
+      skip += 1000;
+    }
+    let dataNow = [...new Map(allPools.map(pool => [pool.id, pool])).values()];
 
     // uni v3 subgraph reserves values are wrong!
     // instead of relying on subgraph values, gonna pull reserve data from contracts
@@ -151,11 +159,16 @@ const topLvl = async (
 
     // pull 24h offset data to calculate fees from swap volume
     let queryPriorC = queryPrior;
-    let dataPrior = await request(
-      url,
-      queryPriorC.replace('<PLACEHOLDER>', blockPrior)
-    );
-    dataPrior = dataPrior.pools;
+    let allPriorPools = [];
+    skip = 0;
+    while (true) {
+      console.log(`allPriorPools: query pools from: ${skip} to: ${skip + 1000}`);
+      let currentData = await request(url, queryPriorC.replace('<PLACEHOLDER>', blockPrior).replace('<SKIP>', skip));
+      if (!currentData.pools || currentData.pools.length === 0) break;
+      allPriorPools = [...allPriorPools, ...currentData.pools];
+      skip += 1000;
+    }
+    let dataPrior = [...new Map(allPriorPools.map(pool => [pool.id, pool])).values()];
 
     // calculate tvl
     dataNow = await utils.tvl(dataNow, chainString);
@@ -179,9 +192,16 @@ const topLvl = async (
     });
 
     // for new v3 apy calc
-    const dataPrior7d = (
-      await request(url, queryPriorC.replace('<PLACEHOLDER>', blockPrior7d))
-    ).pools;
+    let allPrior7dPools = [];
+    skip = 0;
+    while (true) {
+      console.log(`allPrior7dPools: query pools from: ${skip} to: ${skip + 1000}`);
+      let currentData = await request(url, queryPriorC.replace('<PLACEHOLDER>', blockPrior7d).replace('<SKIP>', skip));
+      if (!currentData.pools || currentData.pools.length === 0) break;
+      allPrior7dPools = [...allPrior7dPools, ...currentData.pools];
+      skip += 1000;
+    }
+    const dataPrior7d = [...new Map(allPrior7dPools.map(pool => [pool.id, pool])).values()];
 
     // calc apy (note: old way of using 24h fees * 365 / tvl. keeping this for now) and will store the
     // new apy calc as a separate field
@@ -335,6 +355,12 @@ const topLvl = async (
 };
 
 const main = async (timestamp = null) => {
+  const uniswapV2Pools = new Set(
+    (await axios.get('https://yields.llama.fi/distinctID')).data
+      .filter((p) => p.project === 'uniswap-v2')
+      .map((p) => p.pool)
+  );
+
   const stablecoins = (
     await axios.get(
       'https://stablecoins.llama.fi/stablecoins?includePrices=true'
@@ -359,7 +385,14 @@ const main = async (timestamp = null) => {
           '0x0c6d9d0f82ed2e0b86c4d3e9a9febf95415d1b76',
           '0xc809d13e9ea08f296d3b32d4c69d46ff90f73fd8',
         ].includes(p.pool)
-    );
+    )
+    .filter((p) => {
+      if (uniswapV2Pools.has(p.pool)) {
+        console.log(`Warning: Filtered out duplicate pool ${p.pool} (exists in Uniswap V2)`);
+        return false;
+      }
+      return true;
+    });
 };
 
 module.exports = {