From 4dd1c6ed6fad43b28cdad242c7045403e272e52a Mon Sep 17 00:00:00 2001
From: 0xpeluche <0xpeluche@proton.me>
Date: Fri, 3 Jan 2025 13:36:50 +0100
Subject: [PATCH 1/6] wip

---
 test.js | 114 ++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 95 insertions(+), 19 deletions(-)

diff --git a/test.js b/test.js
index 686d68c1fd..aa59d68f42 100644
--- a/test.js
+++ b/test.js
@@ -1,6 +1,5 @@
 #!/usr/bin/env node
 
-
 const handleError = require('./utils/handleError')
 const INTERNAL_CACHE_FILE = 'tvl-adapter-repo/sdkInternalCache.json'
 process.on('unhandledRejection', handleError)
@@ -31,7 +30,7 @@ if (process.env.LLAMA_SANITIZE)
     delete process.env[key]
   })
 process.env.SKIP_RPC_CHECK = 'true'
-
+const CG_API_KEY = process.env.CG_API_KEY
 
 async function getTvl(
   unixTimestamp,
@@ -403,19 +402,27 @@ async function computeTVL(balances, timestamp) {
   const tokenBalances = {};
   const usdTokenBalances = {};
 
-  tokenData.forEach(response => {
-    Object.keys(response).forEach(address => {
-      delete unknownTokens[address]
+  for (const response of tokenData) {
+    for (const address of Object.keys(response)) {
+      delete unknownTokens[address];
       const data = response[address];
       const balance = balances[address];
-
-      if (data == undefined) tokenBalances[`UNKNOWN (${address})`] = balance
-      if ('confidence' in data && data.confidence < confidenceThreshold || !data.price) return
-      if (Math.abs(data.timestamp - Date.now() / 1e3) > (24 * 3600)) {
-        console.log(`Price for ${address} is stale, ignoring...`)
-        return
+      const symbol = response[address].symbol
+  
+      if (data === undefined) {
+        tokenBalances[`UNKNOWN (${address})`] = balance;
+        continue;
       }
-
+  
+      if (('confidence' in data && data.confidence < confidenceThreshold) || !data.price) {
+        continue;
+      }
+  
+      if (Math.abs(data.timestamp - Date.now() / 1e3) > 24 * 3600) {
+        console.log(`Price for ${address} is stale, ignoring...`);
+        continue;
+      }
+  
       let amount, usdAmount;
       if (address.includes(":") && !address.startsWith("coingecko:")) {
         amount = new BigNumber(balance).div(10 ** data.decimals).toNumber();
@@ -424,21 +431,29 @@ async function computeTVL(balances, timestamp) {
         amount = Number(balance);
         usdAmount = amount * data.price;
       }
-
+  
       if (usdAmount > 1e8) {
         console.log(`-------------------
-Warning: `)
-        console.log(`Token ${address} has more than 100M in value (${usdAmount / 1e6} M) , price data: `, data)
-        console.log(`-------------------`)
+  Warning: `);
+        console.log(`Token ${address} has more than 100M in value (${usdAmount / 1e6} M), price data: `, data);
+
+        const isValidMarketCap = await checkMarketCap(address, symbol, usdAmount);
+        if (!isValidMarketCap) {
+          console.log(`Skipping ${symbol} (${address}) due to invalid market cap. \n`);
+          continue;
+        }
+        console.log(`-------------------`);
       }
+  
       tokenBalances[data.symbol] = (tokenBalances[data.symbol] ?? 0) + amount;
       usdTokenBalances[data.symbol] = (usdTokenBalances[data.symbol] ?? 0) + usdAmount;
       usdTvl += usdAmount;
+  
       if (isNaN(usdTvl)) {
-        throw new Error(`NaN usdTvl for ${address} with balance ${balance} and price ${data.price}`)
+        throw new Error(`NaN usdTvl for ${address} with balance ${balance} and price ${data.price}`);
       }
-    })
-  });
+    }
+  }
 
   Object.keys(unknownTokens).forEach(address => tokenBalances[`UNKNOWN (${address})`] = balances[address])
 
@@ -477,6 +492,67 @@ function buildPricesGetQueries(readKeys) {
   return queries
 }
 
+async function checkMarketCap(address, symbol, usdAmount) {
+  console.log(`Checking the Market Cap for ${address} (${symbol})`);
+
+  if (!CG_API_KEY) {
+    console.log(`⚠️  CG_API_KEY is not set, skipping check`);
+    return true;
+  }
+
+  const [coingeckoId] = await searchCoingeckoId(symbol);
+
+  if (!coingeckoId) {
+    console.log(`⚠️  No CoinGecko ID found for symbol: ${symbol}`);
+    console.log(`⚠️  Proceeding cautiously and counting the token.`);
+    return true;
+  }
+
+  const coinMarketCap = await fetchCoinMarketCap(coingeckoId);
+
+  if (!coinMarketCap) {
+    console.log(`⚠️  No market cap found for CoinGecko ID: ${coingeckoId}`);
+    console.log(`⚠️  Proceeding cautiously and counting the token.`);
+    return true;
+  }
+
+  console.log(`✅ Market Cap for ${symbol} (${coingeckoId}) found: ${coinMarketCap}`);
+
+  if (usdAmount > coinMarketCap) {
+    console.log(`❌ USD amount (${usdAmount}) exceeds market cap (${coinMarketCap}) for ${symbol}. Ignoring token.`);
+    return false;
+  }
+
+  console.log(`✅ USD amount (${usdAmount}) is within market cap (${coinMarketCap}) for ${symbol}. Counting token.`);
+  return true;
+}
+
+async function searchCoingeckoId(symbol) {
+  const url = `https://pro-api.coingecko.com/api/v3/search?query=${symbol}`;
+  const { data } = await axios.get(url, {
+    headers: {
+      accept: 'application/json',
+      'x-cg-pro-api-key': CG_API_KEY,
+    },
+  });
+
+  return data.coins
+  .filter((coin) => coin.symbol.toLowerCase() === symbol.toLowerCase())
+  .map((coin) => coin.id);
+}
+
+async function fetchCoinMarketCap(id){
+  const url = `https://pro-api.coingecko.com/api/v3/coins/${id}?localization=false&tickers=true&market_data=true&community_data=false&developer_data=false&sparkline=false`;
+  const { data } = await axios.get(url, {
+    headers: {
+      accept: 'application/json',
+      'x-cg-pro-api-key': CG_API_KEY
+    }
+  });
+
+  return data.market_data.market_cap.usd
+}
+
 async function initCache() {
   let currentCache = await sdk.cache.readCache(INTERNAL_CACHE_FILE)
   // if (process.env.NO_EXIT_ON_LONG_RUN_RPC)

From 09cf788353e3d1627dd4c5f7d274951f7e33dba7 Mon Sep 17 00:00:00 2001
From: 0xpeluche <0xpeluche@proton.me>
Date: Mon, 6 Jan 2025 10:56:38 +0100
Subject: [PATCH 2/6] Check mCap

---
 test.js | 81 ++++++++++++++++++++++++++++-----------------------------
 1 file changed, 40 insertions(+), 41 deletions(-)

diff --git a/test.js b/test.js
index aa59d68f42..d34877bc8c 100644
--- a/test.js
+++ b/test.js
@@ -436,12 +436,7 @@ async function computeTVL(balances, timestamp) {
         console.log(`-------------------
   Warning: `);
         console.log(`Token ${address} has more than 100M in value (${usdAmount / 1e6} M), price data: `, data);
-
-        const isValidMarketCap = await checkMarketCap(address, symbol, usdAmount);
-        if (!isValidMarketCap) {
-          console.log(`Skipping ${symbol} (${address}) due to invalid market cap. \n`);
-          continue;
-        }
+        await checkMarketCap(address, symbol, usdAmount);
         console.log(`-------------------`);
       }
   
@@ -492,32 +487,25 @@ function buildPricesGetQueries(readKeys) {
   return queries
 }
 
-async function checkMarketCap(address, symbol, usdAmount) {
-  console.log(`Checking the Market Cap for ${address} (${symbol})`);
+async function checkMarketCap(token, symbol, usdAmount) {
+  console.log(`Checking the Market Cap for ${token} (${symbol})`);
 
   if (!CG_API_KEY) {
     console.log(`⚠️  CG_API_KEY is not set, skipping check`);
     return true;
   }
 
-  const [coingeckoId] = await searchCoingeckoId(symbol);
-
-  if (!coingeckoId) {
-    console.log(`⚠️  No CoinGecko ID found for symbol: ${symbol}`);
-    console.log(`⚠️  Proceeding cautiously and counting the token.`);
-    return true;
-  }
+  const normalizedToken = symbol === "ETH" ? "coingecko:ethereum" : token;
 
-  const coinMarketCap = await fetchCoinMarketCap(coingeckoId);
+  const coinMarketCap = normalizedToken.includes(":") && !normalizedToken.startsWith("coingecko:")
+    ? await fetchCoinMarketCap(normalizedToken, symbol, true)
+    : await fetchCoinMarketCap(normalizedToken, symbol, false);
 
-  if (!coinMarketCap) {
-    console.log(`⚠️  No market cap found for CoinGecko ID: ${coingeckoId}`);
-    console.log(`⚠️  Proceeding cautiously and counting the token.`);
+  if (coinMarketCap === null) {
+    console.log(`⚠️  No market cap found for ${symbol}. Proceeding cautiously.`);
     return true;
   }
 
-  console.log(`✅ Market Cap for ${symbol} (${coingeckoId}) found: ${coinMarketCap}`);
-
   if (usdAmount > coinMarketCap) {
     console.log(`❌ USD amount (${usdAmount}) exceeds market cap (${coinMarketCap}) for ${symbol}. Ignoring token.`);
     return false;
@@ -527,30 +515,41 @@ async function checkMarketCap(address, symbol, usdAmount) {
   return true;
 }
 
-async function searchCoingeckoId(symbol) {
-  const url = `https://pro-api.coingecko.com/api/v3/search?query=${symbol}`;
-  const { data } = await axios.get(url, {
-    headers: {
-      accept: 'application/json',
-      'x-cg-pro-api-key': CG_API_KEY,
-    },
-  });
+async function fetchCoinMarketCap(token, symbol, isByChain) {
+  try {
+    const [chain, address] = isByChain ? token.split(":") : ["", token.split(":")[1]];
+    const url = isByChain
+      ? `https://pro-api.coingecko.com/api/v3/coins/${chain}/contract/${address}`
+      : `https://pro-api.coingecko.com/api/v3/coins/${address}?localization=false&tickers=true&market_data=true&community_data=false&developer_data=false&sparkline=false`;
+
+    const { data } = await axios.get(url, {
+      headers: {
+        accept: "application/json",
+        "x-cg-pro-api-key": CG_API_KEY,
+      },
+    });
 
-  return data.coins
-  .filter((coin) => coin.symbol.toLowerCase() === symbol.toLowerCase())
-  .map((coin) => coin.id);
-}
+    if (!data) {
+      console.log(`❌ No data returned from CoinGecko API for ${token}`);
+      return null;
+    }
 
-async function fetchCoinMarketCap(id){
-  const url = `https://pro-api.coingecko.com/api/v3/coins/${id}?localization=false&tickers=true&market_data=true&community_data=false&developer_data=false&sparkline=false`;
-  const { data } = await axios.get(url, {
-    headers: {
-      accept: 'application/json',
-      'x-cg-pro-api-key': CG_API_KEY
+    if (!data.symbol || data.symbol.toLowerCase() !== symbol.toLowerCase()) {
+      console.log(`❌ Symbol mismatch for ${token} (expected: ${symbol}, received: ${data.symbol || "none"})`);
+      return null;
     }
-  });
 
-  return data.market_data.market_cap.usd
+    if (!data.market_data || !data.market_data.market_cap || !data.market_data.market_cap.usd) {
+      console.log(`❌ No market cap data found for ${token}`);
+      return null;
+    }
+
+    console.log(`✅ Market cap found for ${symbol}: ${data.market_data.market_cap.usd} USD`);
+    return data.market_data.market_cap.usd;
+  } catch (error) {
+    console.error(`❌ Error while querying CoinGecko API for ${token}:`, error.message);
+    return null;
+  }
 }
 
 async function initCache() {

From 3597fc740a4e7d3ad8b2275eabe0811531070ece Mon Sep 17 00:00:00 2001
From: 0xpeluche <0xpeluche@proton.me>
Date: Mon, 6 Jan 2025 11:19:48 +0100
Subject: [PATCH 3/6] .

---
 test.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test.js b/test.js
index d34877bc8c..77086f8165 100644
--- a/test.js
+++ b/test.js
@@ -436,7 +436,12 @@ async function computeTVL(balances, timestamp) {
         console.log(`-------------------
   Warning: `);
         console.log(`Token ${address} has more than 100M in value (${usdAmount / 1e6} M), price data: `, data);
-        await checkMarketCap(address, symbol, usdAmount);
+
+        const isWithinMarketCap = await checkMarketCap(address, symbol, usdAmount);
+        if (!isWithinMarketCap) {
+          console.log(`Ignoring token ${address} (${symbol}) due to exceeding market cap.`);
+          continue;
+        }
         console.log(`-------------------`);
       }
   

From ad615abfe0d2cd339a04f9fc6e1885c740fc8f04 Mon Sep 17 00:00:00 2001
From: 0xpeluche <0xpeluche@proton.me>
Date: Mon, 6 Jan 2025 11:35:06 +0100
Subject: [PATCH 4/6] using new price if possible

---
 test.js | 53 +++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 37 insertions(+), 16 deletions(-)

diff --git a/test.js b/test.js
index 77086f8165..f5a348c334 100644
--- a/test.js
+++ b/test.js
@@ -437,10 +437,22 @@ async function computeTVL(balances, timestamp) {
   Warning: `);
         console.log(`Token ${address} has more than 100M in value (${usdAmount / 1e6} M), price data: `, data);
 
-        const isWithinMarketCap = await checkMarketCap(address, symbol, usdAmount);
+        const { isWithinMarketCap, newPrice } = await checkMarketCap(address, symbol, usdAmount);
+
         if (!isWithinMarketCap) {
-          console.log(`Ignoring token ${address} (${symbol}) due to exceeding market cap.`);
-          continue;
+          if (!newPrice) {
+            console.log(`Token ${address} (${symbol}) exceeds market cap and no valid price was found. Ignoring token.`);
+            continue;
+          }
+        
+          console.log(`⚠️  Token ${address} (${symbol}) exceeds market cap. Using updated price from CoinGecko: ${newPrice}`);
+          data.price = newPrice;
+        
+          amount = address.includes(":") && !address.startsWith("coingecko:")
+            ? new BigNumber(balance).div(10 ** data.decimals).toNumber()
+            : Number(balance);
+        
+          usdAmount = amount * data.price;
         }
         console.log(`-------------------`);
       }
@@ -497,27 +509,33 @@ async function checkMarketCap(token, symbol, usdAmount) {
 
   if (!CG_API_KEY) {
     console.log(`⚠️  CG_API_KEY is not set, skipping check`);
-    return true;
+    return { isWithinMarketCap: true, newPrice: null };
   }
 
   const normalizedToken = symbol === "ETH" ? "coingecko:ethereum" : token;
 
-  const coinMarketCap = normalizedToken.includes(":") && !normalizedToken.startsWith("coingecko:")
+  const coinData = normalizedToken.includes(":") && !normalizedToken.startsWith("coingecko:")
     ? await fetchCoinMarketCap(normalizedToken, symbol, true)
     : await fetchCoinMarketCap(normalizedToken, symbol, false);
 
-  if (coinMarketCap === null) {
+  if (!coinData || !coinData.marketCap) {
     console.log(`⚠️  No market cap found for ${symbol}. Proceeding cautiously.`);
-    return true;
+    return { isWithinMarketCap: true, newPrice: null };
   }
 
-  if (usdAmount > coinMarketCap) {
-    console.log(`❌ USD amount (${usdAmount}) exceeds market cap (${coinMarketCap}) for ${symbol}. Ignoring token.`);
-    return false;
+  const { marketCap, price } = coinData;
+
+  if (usdAmount > marketCap) {
+    if (!price) {
+      console.log(`❌ USD amount (${usdAmount}) exceeds market cap (${marketCap}), and no valid price found on CoinGecko for ${symbol}. Ignoring token.`);
+      return { isWithinMarketCap: false, newPrice: null };
+    }
+    console.log(`❌ USD amount (${usdAmount}) exceeds market cap (${marketCap}) for ${symbol}. Using updated price from CoinGecko.`);
+    return { isWithinMarketCap: false, newPrice: price };
   }
 
-  console.log(`✅ USD amount (${usdAmount}) is within market cap (${coinMarketCap}) for ${symbol}. Counting token.`);
-  return true;
+  console.log(`✅ USD amount (${usdAmount}) is within market cap (${marketCap}) for ${symbol}. Counting token.`);
+  return { isWithinMarketCap: true, newPrice: null };
 }
 
 async function fetchCoinMarketCap(token, symbol, isByChain) {
@@ -544,13 +562,16 @@ async function fetchCoinMarketCap(token, symbol, isByChain) {
       return null;
     }
 
-    if (!data.market_data || !data.market_data.market_cap || !data.market_data.market_cap.usd) {
+    const marketCap = data.market_data?.market_cap?.usd || null;
+    const price = data.market_data?.current_price?.usd || null;
+
+    if (!marketCap) {
       console.log(`❌ No market cap data found for ${token}`);
-      return null;
+    } else {
+      console.log(`✅ Market cap found for ${symbol}: ${marketCap} USD`);
     }
 
-    console.log(`✅ Market cap found for ${symbol}: ${data.market_data.market_cap.usd} USD`);
-    return data.market_data.market_cap.usd;
+    return { marketCap, price };
   } catch (error) {
     console.error(`❌ Error while querying CoinGecko API for ${token}:`, error.message);
     return null;

From 6093e811b664f04cb4cf06afae5b9f0e09301089 Mon Sep 17 00:00:00 2001
From: 0xpeluche <0xpeluche@proton.me>
Date: Mon, 6 Jan 2025 11:36:44 +0100
Subject: [PATCH 5/6] .

---
 test.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test.js b/test.js
index f5a348c334..1f92a8886f 100644
--- a/test.js
+++ b/test.js
@@ -441,7 +441,7 @@ async function computeTVL(balances, timestamp) {
 
         if (!isWithinMarketCap) {
           if (!newPrice) {
-            console.log(`Token ${address} (${symbol}) exceeds market cap and no valid price was found. Ignoring token.`);
+            console.log(`❌ Token ${address} (${symbol}) exceeds market cap and no valid price was found. Ignoring token.`);
             continue;
           }
         

From ce0725ce7b2fe09a191f737a3e5df68c7385e88b Mon Sep 17 00:00:00 2001
From: 0xpeluche <0xpeluche@proton.me>
Date: Mon, 6 Jan 2025 11:54:02 +0100
Subject: [PATCH 6/6] check with new price

---
 test.js | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/test.js b/test.js
index 1f92a8886f..4cd2b1458d 100644
--- a/test.js
+++ b/test.js
@@ -437,7 +437,7 @@ async function computeTVL(balances, timestamp) {
   Warning: `);
         console.log(`Token ${address} has more than 100M in value (${usdAmount / 1e6} M), price data: `, data);
 
-        const { isWithinMarketCap, newPrice } = await checkMarketCap(address, symbol, usdAmount);
+        const { isWithinMarketCap, newPrice, marketCap } = await checkMarketCap(address, symbol, usdAmount);
 
         if (!isWithinMarketCap) {
           if (!newPrice) {
@@ -453,6 +453,13 @@ async function computeTVL(balances, timestamp) {
             : Number(balance);
         
           usdAmount = amount * data.price;
+
+          if (usdAmount > marketCap) {
+            console.log(`❌ Even with updated price, USD amount (${usdAmount}) still exceeds market cap (${marketCap}). Ignoring token.`);
+            continue;
+          }
+
+          console.log(`✅ With updated price, USD amount (${usdAmount}) is now within market cap (${marketCap}). Counting token.`);
         }
         console.log(`-------------------`);
       }
@@ -509,7 +516,7 @@ async function checkMarketCap(token, symbol, usdAmount) {
 
   if (!CG_API_KEY) {
     console.log(`⚠️  CG_API_KEY is not set, skipping check`);
-    return { isWithinMarketCap: true, newPrice: null };
+    return { isWithinMarketCap: true, newPrice: null, marketCap: null };
   }
 
   const normalizedToken = symbol === "ETH" ? "coingecko:ethereum" : token;
@@ -520,7 +527,7 @@ async function checkMarketCap(token, symbol, usdAmount) {
 
   if (!coinData || !coinData.marketCap) {
     console.log(`⚠️  No market cap found for ${symbol}. Proceeding cautiously.`);
-    return { isWithinMarketCap: true, newPrice: null };
+    return { isWithinMarketCap: true, newPrice: null, marketCap: null };
   }
 
   const { marketCap, price } = coinData;
@@ -528,14 +535,15 @@ async function checkMarketCap(token, symbol, usdAmount) {
   if (usdAmount > marketCap) {
     if (!price) {
       console.log(`❌ USD amount (${usdAmount}) exceeds market cap (${marketCap}), and no valid price found on CoinGecko for ${symbol}. Ignoring token.`);
-      return { isWithinMarketCap: false, newPrice: null };
+      return { isWithinMarketCap: false, newPrice: null, marketCap };
     }
-    console.log(`❌ USD amount (${usdAmount}) exceeds market cap (${marketCap}) for ${symbol}. Using updated price from CoinGecko.`);
-    return { isWithinMarketCap: false, newPrice: price };
+
+    console.log(`❌ USD amount (${usdAmount}) exceeds market cap (${marketCap}). Using updated price from CoinGecko: ${price}`);
+    return { isWithinMarketCap: false, newPrice: price, marketCap };
   }
 
   console.log(`✅ USD amount (${usdAmount}) is within market cap (${marketCap}) for ${symbol}. Counting token.`);
-  return { isWithinMarketCap: true, newPrice: null };
+  return { isWithinMarketCap: true, newPrice: null, marketCap };
 }
 
 async function fetchCoinMarketCap(token, symbol, isByChain) {