Merge pull request #28 from golemfactory/scx-scanner-payments

New payments collection.

stats-backend/api/urls.py

@@ -12,6 +12,7 @@
         views.payments_last_n_hours_provider,
     ),
     path("provider/node/<yagna_id>/earnings", views.payments_earnings_provider),
+    path("provider/node/<yagna_id>/earningsnew", views.payments_earnings_provider_new),
     path("provider/node/<yagna_id>/activity", views.activity_graph_provider),
     path("provider/node/<yagna_id>/total/computed", views.total_tasks_computed),
     path(
@@ -25,6 +26,7 @@
     path("network/earnings/24", views.network_earnings_24h),
     path("network/earnings/90d", views.network_total_earnings),
     path("network/earnings/overview", views.network_earnings_overview),
+    path("network/earnings/overviewnew", views.network_earnings_overview_new),
     path("network/online", views.online_nodes),
     path("network/online/stats", views.general_stats),
     path("network/utilization", views.network_utilization),

stats-backend/api/views.py

@@ -11,6 +11,7 @@
     NetworkStats,
     ProvidersComputing,
     Benchmark,
+    Requestors,
 )
 from .models import APICounter
 from .serializers import (
@@ -224,6 +225,11 @@ async def yagna_releases(request):
         return HttpResponse(status=400)
 
 
+import requests
+
+from api2.models import RelayNodes
+
+
 async def payments_earnings_provider(request, yagna_id):
     now = round(time.time())
     time_intervals = ["24", "168", "720", "2160"]
@@ -244,6 +250,54 @@ async def payments_earnings_provider(request, yagna_id):
     return JsonResponse(earnings, json_dumps_params={"indent": 4})
 
 
+def payments_earnings_provider_new(request, yagna_id):
+    now = int(time.time())
+    hour_intervals = [24, 168, 720, 2160]
+    base_url = "http://erc20-api/erc20/api/stats/transfers?chain=137&receiver="
+
+    earnings = {}
+
+    for interval in hour_intervals:
+        epoch = now - (interval * 3600)
+        url = f"{base_url}{yagna_id}&from={epoch}&to={now}"
+        response = requests.get(url)
+
+        if response.status_code == 200:
+            data = response.json()
+            transfers = data.get("transfers", [])
+
+            from_addrs = {t["fromAddr"] for t in transfers}
+            matched_addrs = set(
+                Requestors.objects.filter(node_id__in=from_addrs).values_list(
+                    "node_id", flat=True
+                )
+            ).union(
+                set(
+                    RelayNodes.objects.filter(node_id__in=from_addrs).values_list(
+                        "node_id", flat=True
+                    )
+                )
+            )
+
+            total_amount_wei_matched = 0
+            for t in transfers:
+                if t["fromAddr"] in matched_addrs:
+                    total_amount_wei_matched += int(t["tokenAmount"])
+
+            earnings[str(interval)] = total_amount_wei_matched / 1e18
+        else:
+            earnings[str(interval)] = 0.0
+        total_earnings = (
+            GolemTransactions.objects.filter(
+                receiver=yagna_id, tx_from_golem=True
+            ).aggregate(Sum("amount"))["amount__sum"]
+            or 0.0
+        )
+        earnings["total"] = total_earnings
+
+    return JsonResponse(earnings, json_dumps_params={"indent": 4})
+
+
 async def total_tasks_computed(request, yagna_id):
     now = round(time.time())
     domain = (
@@ -655,6 +709,13 @@ async def network_earnings_6h(request):
         return HttpResponse(status=400)
 
 
+from api2.models import GolemTransactions
+from django.db.models import Sum
+
+from django.utils.timezone import now
+from datetime import timedelta
+
+
 async def network_earnings_overview(request):
     """
     Returns the earnings for the whole network over time for various time frames,
@@ -691,6 +752,41 @@ async def network_earnings_overview(request):
         return HttpResponse(status=400)
 
 
+def network_earnings_overview_new(request):
+    if request.method == "GET":
+        time_frames = [6, 24, 168, 720, 2160]
+        response_data = {}
+
+        for frame in time_frames:
+            end_time = now()
+            start_time = end_time - timedelta(hours=frame)
+            total_earnings = (
+                GolemTransactions.objects.filter(
+                    timestamp__range=[start_time, end_time], tx_from_golem=True
+                ).aggregate(Sum("amount"))["amount__sum"]
+                or 0.0
+            )
+
+            response_data[f"network_earnings_{frame}h"] = {
+                "total_earnings": float(total_earnings)
+            }
+
+        all_time_earnings = (
+            GolemTransactions.objects.filter(tx_from_golem=True).aggregate(
+                Sum("amount")
+            )["amount__sum"]
+            or 0.0
+        )
+
+        print(f"all_time_earnings: {all_time_earnings}")
+
+        response_data["network_total_earnings"] = {
+            "total_earnings": float(all_time_earnings)
+        }
+
+        return JsonResponse(response_data)
+
+
 async def requestors(request):
     """
     Returns all the requestors seen on the network and the tasks requested amount.

stats-backend/api2/migrations/0024_golemtransactions_transactionscraperindex.py

@@ -0,0 +1,34 @@
+# Generated by Django 4.1.7 on 2024-03-20 23:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api2', '0023_relaynodes'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GolemTransactions',
+            fields=[
+                ('scanner_id', models.IntegerField(primary_key=True, serialize=False, unique=True)),
+                ('txhash', models.CharField(max_length=66)),
+                ('transaction_type', models.CharField(blank=True, max_length=42, null=True)),
+                ('amount', models.FloatField()),
+                ('timestamp', models.DateTimeField()),
+                ('receiver', models.CharField(max_length=42)),
+                ('sender', models.CharField(max_length=42)),
+                ('tx_from_golem', models.BooleanField(default=False)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='TransactionScraperIndex',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('indexed_before', models.BooleanField(default=False)),
+                ('latest_timestamp_indexed', models.DateTimeField(blank=True, null=True)),
+            ],
+        ),
+    ]

stats-backend/api2/migrations/0025_alter_golemtransactions_receiver_and_more.py

@@ -0,0 +1,64 @@
+# Generated by Django 4.1.7 on 2024-03-20 23:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api2', '0024_golemtransactions_transactionscraperindex'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='golemtransactions',
+            name='receiver',
+            field=models.CharField(db_index=True, max_length=42),
+        ),
+        migrations.AlterField(
+            model_name='golemtransactions',
+            name='scanner_id',
+            field=models.IntegerField(primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='golemtransactions',
+            name='sender',
+            field=models.CharField(db_index=True, max_length=42),
+        ),
+        migrations.AlterField(
+            model_name='golemtransactions',
+            name='timestamp',
+            field=models.DateTimeField(db_index=True),
+        ),
+        migrations.AlterField(
+            model_name='golemtransactions',
+            name='transaction_type',
+            field=models.CharField(blank=True, db_index=True, max_length=42, null=True),
+        ),
+        migrations.AlterField(
+            model_name='golemtransactions',
+            name='tx_from_golem',
+            field=models.BooleanField(db_index=True, default=False),
+        ),
+        migrations.AlterField(
+            model_name='golemtransactions',
+            name='txhash',
+            field=models.CharField(db_index=True, max_length=66),
+        ),
+        migrations.AddIndex(
+            model_name='golemtransactions',
+            index=models.Index(fields=['txhash'], name='api2_golemt_txhash_205c12_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='golemtransactions',
+            index=models.Index(fields=['transaction_type'], name='api2_golemt_transac_b601e3_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='golemtransactions',
+            index=models.Index(fields=['timestamp'], name='api2_golemt_timesta_f4c9b4_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='golemtransactions',
+            index=models.Index(fields=['receiver', 'sender'], name='api2_golemt_receive_d6425b_idx'),
+        ),
+    ]

stats-backend/api2/models.py

@@ -114,3 +114,32 @@ class PricingSnapshot(models.Model):
 
 class RelayNodes(models.Model):
     node_id = models.CharField(max_length=42, unique=True)
+
+
+
+class GolemTransactions(models.Model):
+    scanner_id = models.IntegerField(primary_key=True)
+    txhash = models.CharField(max_length=66, db_index=True)
+    transaction_type = models.CharField(
+        max_length=42, null=True, blank=True, db_index=True
+    )
+    amount = models.FloatField()
+    timestamp = models.DateTimeField(db_index=True)
+    receiver = models.CharField(max_length=42, db_index=True)
+    sender = models.CharField(max_length=42, db_index=True)
+    tx_from_golem = models.BooleanField(default=False, db_index=True)
+
+    class Meta:
+        indexes = [
+            models.Index(fields=["txhash"]),
+            models.Index(fields=["transaction_type"]),
+            models.Index(fields=["timestamp"]),
+            models.Index(fields=["receiver", "sender"]),
+            # Compound index example if you often filter by both receiver and sender
+        ]
+
+
+class TransactionScraperIndex(models.Model):
+    indexed_before = models.BooleanField(default=False)
+    latest_timestamp_indexed = models.DateTimeField(null=True, blank=True)
+