diff --git a/backend/app.py b/backend/app.py index f2c64c81..3218f80d 100644 --- a/backend/app.py +++ b/backend/app.py @@ -15,6 +15,7 @@ from routes.submit_room_line import submit_room_line_bp from routes.admin import admin_bp from routes.frontend import frontend_bp +from routes.analytics import analytics_bp from services.db import redis_client from services.canvas_counter import get_canvas_draw_count from services.graphql_service import commit_transaction_via_graphql @@ -143,6 +144,7 @@ def handle_all_exceptions(e): # Frontend serving must be last to avoid route conflicts app.register_blueprint(frontend_bp) +app.register_blueprint(analytics_bp) if __name__ == '__main__': if not redis_client.exists('res-canvas-draw-count'): diff --git a/backend/config.py b/backend/config.py index 827d2346..db3c2777 100644 --- a/backend/config.py +++ b/backend/config.py @@ -19,6 +19,12 @@ LOG_FILE = "backend_graphql.log" +# Analytics / LLM configuration +ANALYTICS_ENABLED = os.getenv("ANALYTICS_ENABLED", "True") == "True" +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") +ANALYTICS_COLLECTION_NAME = os.getenv("ANALYTICS_COLLECTION_NAME", "analytics_events") +ANALYTICS_AGGREGATES_COLLECTION = os.getenv("ANALYTICS_AGGREGATES_COLLECTION", "analytics_aggregates") + JWT_SECRET = os.getenv("JWT_SECRET", "dev-insecure-change-me") JWT_ISSUER = "rescanvas" diff --git a/backend/routes/analytics.py b/backend/routes/analytics.py new file mode 100644 index 00000000..9355f1e5 --- /dev/null +++ b/backend/routes/analytics.py @@ -0,0 +1,49 @@ +from flask import Blueprint, request, jsonify +from services.db import analytics_aggregates_coll +from services.analytics_service import query_recent +from services.insights_generator import generate_insights +import logging + +analytics_bp = Blueprint('analytics_bp', __name__) +logger = logging.getLogger(__name__) + + +@analytics_bp.route('/api/analytics/recent', methods=['GET']) +def recent_events(): + room = request.args.get('roomId') + limit = int(request.args.get('limit', '100')) + docs = query_recent(room, limit=limit) + # convert ObjectId to str if needed + def clean(d): + d = dict(d) + d.pop('_id', None) + return d + return jsonify([clean(d) for d in docs]) + + +@analytics_bp.route('/api/analytics/overview', methods=['GET']) +def overview(): + room = request.args.get('roomId') + q = {} if not room else {"roomId": str(room)} + ag = analytics_aggregates_coll.find_one(q) or {} + ag.pop('_id', None) + return jsonify(ag) + + +@analytics_bp.route('/api/analytics/insights', methods=['POST']) +def insights(): + data = request.get_json() or {} + room = data.get('roomId') + q = {} if not room else {"roomId": str(room)} + ag = analytics_aggregates_coll.find_one(q) or {} + try: + res = generate_insights(ag) + return jsonify(res) + except Exception: + logger.exception('Failed to generate insights') + return jsonify({"error": "failed"}), 500 + + +@analytics_bp.route('/api/analytics/health', methods=['GET']) +def health(): + return jsonify({"ok": True}) diff --git a/backend/routes/socketio_handlers.py b/backend/routes/socketio_handlers.py index e34bf018..b6a79018 100644 --- a/backend/routes/socketio_handlers.py +++ b/backend/routes/socketio_handlers.py @@ -3,6 +3,7 @@ from flask_socketio import join_room, leave_room, emit from services.socketio import socketio from services.db import rooms_coll, shares_coll, users_coll +from services.analytics_service import ingest_event import logging from config import JWT_SECRET import jwt @@ -122,6 +123,16 @@ def on_join_room(data): payload = {'roomId': room_id, 'userId': user_id, 'username': username_to_emit, 'members': members} logging.getLogger(__name__).info('socket: emitting user_joined to room %s payload=%s sid=%s had_cached_claims=%s', room_id, payload, sid, had_cached) emit('user_joined', payload, room=f"room:{room_id}") + try: + # record join event for analytics (anonymized inside service) + ingest_event({ + 'roomId': room_id, + 'userId': user_id, + 'eventType': 'join', + 'payload': {'username': username_to_emit} + }) + except Exception: + pass try: emit('server_debug', {'action': 'emitted_user_joined', 'sid': sid, 'had_cached_claims': had_cached, 'payload': payload}, room=None) except Exception: @@ -150,5 +161,14 @@ def on_leave_room(data): payload = {'roomId': room_id, 'username': username_to_emit, 'members': members} logging.getLogger(__name__).info('socket: emitting user_left to room %s payload=%s', room_id, payload) emit('user_left', payload, room=f"room:{room_id}") + try: + ingest_event({ + 'roomId': room_id, + 'userId': None, # leave events can be anonymous + 'eventType': 'leave', + 'payload': {'username': username_to_emit} + }) + except Exception: + pass except Exception: pass \ No newline at end of file diff --git a/backend/routes/submit_room_line.py b/backend/routes/submit_room_line.py index 265b4912..186c588f 100644 --- a/backend/routes/submit_room_line.py +++ b/backend/routes/submit_room_line.py @@ -6,6 +6,7 @@ from services.graphql_service import commit_transaction_via_graphql from services.db import redis_client, strokes_coll, rooms_coll, shares_coll from services.socketio_service import push_to_room +from services.analytics_service import ingest_event from services.canvas_counter import get_canvas_draw_count, increment_canvas_draw_count from services.crypto_service import unwrap_room_key, encrypt_for_room, wrap_room_key import nacl.signing, nacl.encoding @@ -157,6 +158,20 @@ def submit_room_line(): 'type': room_type }) + try: + ingest_event({ + 'roomId': roomId, + 'userId': actor_id or drawing.get('user'), + 'eventType': 'stroke_created', + 'payload': { + 'color': drawing.get('color'), + 'lineWidth': drawing.get('lineWidth') + }, + 'ts': drawing['timestamp'] + }) + except Exception: + pass + # Update room's updatedAt so the Dashboard's "Last edited" reflects drawing activity try: rooms_coll.update_one({'_id': room['_id']}, {'$set': {'updatedAt': datetime.utcnow()}}) @@ -180,6 +195,20 @@ def submit_room_line(): 'type': 'public' }) + try: + ingest_event({ + 'roomId': roomId, + 'userId': actor_id or drawing.get('user'), + 'eventType': 'stroke_created', + 'payload': { + 'color': drawing.get('color'), + 'lineWidth': drawing.get('lineWidth') + }, + 'ts': drawing['timestamp'] + }) + except Exception: + pass + try: rooms_coll.update_one({'_id': room['_id']}, {'$set': {'updatedAt': datetime.utcnow()}}) except Exception: diff --git a/backend/services/analytics_service.py b/backend/services/analytics_service.py new file mode 100644 index 00000000..07ae8e5a --- /dev/null +++ b/backend/services/analytics_service.py @@ -0,0 +1,65 @@ +""" +Simple analytics ingestion and query helpers. +This module stores anonymized events into MongoDB and provides small helpers +used by socket handlers and the aggregation worker. +""" +import time +import logging +from services.db import analytics_coll +from bson.objectid import ObjectId +from config import ANALYTICS_ENABLED + +logger = logging.getLogger(__name__) + + +def _anonymize_user(user_id): + """Return a deterministic anonymized user id for privacy-aware aggregation.""" + if not user_id: + return None + # Keep short hash-like form + try: + import hashlib + return hashlib.sha1(str(user_id).encode('utf-8')).hexdigest()[:16] + except Exception: + return str(user_id) + + +def ingest_event(event: dict): + """Ingest an analytics event into the analytics collection. + + Event shape (examples): + { + "roomId": "...", + "userId": "...", + "eventType": "stroke_created" | "join" | "leave" | "heartbeat", + "payload": { ... }, + "ts": 1234567890 + } + """ + if not ANALYTICS_ENABLED: + return None + try: + ev = dict(event) + ev.setdefault('ts', int(time.time() * 1000)) + # anonymize userId for privacy + if 'userId' in ev and ev['userId']: + ev['anonUserId'] = _anonymize_user(ev.get('userId')) + ev.pop('userId', None) + # ensure roomId string + if 'roomId' in ev and isinstance(ev['roomId'], ObjectId): + ev['roomId'] = str(ev['roomId']) + analytics_coll.insert_one(ev) + return ev + except Exception: + logger.exception('Failed to ingest analytics event') + return None + + +def query_recent(roomId=None, limit=100): + q = {} if not roomId else {"roomId": str(roomId)} + try: + docs = list(analytics_coll.find(q).sort([('ts', -1)]).limit(limit)) + return docs + except Exception: + logger.exception('Failed to query recent analytics events') + return [] diff --git a/backend/services/db.py b/backend/services/db.py index b05aee3d..95d189dd 100644 --- a/backend/services/db.py +++ b/backend/services/db.py @@ -39,6 +39,16 @@ invites_coll = mongo_client[DB_NAME]["room_invites"] notifications_coll = mongo_client[DB_NAME]["notifications"] +# Analytics collections +try: + from config import ANALYTICS_COLLECTION_NAME, ANALYTICS_AGGREGATES_COLLECTION +except Exception: + ANALYTICS_COLLECTION_NAME = "analytics_events" + ANALYTICS_AGGREGATES_COLLECTION = "analytics_aggregates" + +analytics_coll = mongo_client[DB_NAME][ANALYTICS_COLLECTION_NAME] +analytics_aggregates_coll = mongo_client[DB_NAME][ANALYTICS_AGGREGATES_COLLECTION] + # TTL index on refresh token expiresAt so expired refresh tokens are removed automatically try: refresh_tokens_coll.create_index("expiresAt", expireAfterSeconds=0) @@ -61,4 +71,10 @@ users_coll.create_index("username", unique=True) rooms_coll.create_index([("ownerId", 1), ("type", 1)]) shares_coll.create_index([("roomId", 1), ("userId", 1)], unique=True) -strokes_coll.create_index([("roomId", 1), ("ts", 1)]) \ No newline at end of file +strokes_coll.create_index([("roomId", 1), ("ts", 1)]) +try: + analytics_coll.create_index([("roomId", 1), ("ts", 1)]) + analytics_coll.create_index([("eventType", 1)]) + analytics_aggregates_coll.create_index([("roomId", 1)]) +except Exception: + pass \ No newline at end of file diff --git a/backend/services/insights_generator.py b/backend/services/insights_generator.py new file mode 100644 index 00000000..5f6d648f --- /dev/null +++ b/backend/services/insights_generator.py @@ -0,0 +1,74 @@ +""" +LLM-based insights generator. Uses OpenAI if API key is available; otherwise +returns simple rule-based summaries. +""" +import logging +from config import OPENAI_API_KEY +import json + +logger = logging.getLogger(__name__) + + +def _summarize_aggregates(aggregates: dict): + # Build a concise prompt from aggregates + try: + total_strokes = aggregates.get('total_strokes', 0) + active_users = aggregates.get('active_users', 0) + top_colors = aggregates.get('top_colors', []) + collab_pairs = aggregates.get('collaboration_pairs', []) + + summary = ( + f"Total strokes: {total_strokes}. Active users: {active_users}. " + f"Top colors: {', '.join(top_colors[:5]) if top_colors else 'N/A'}. " + f"Top collaboration pairs: {', '.join([f'{p[0]}-{p[1]}' for p in collab_pairs[:5]]) if collab_pairs else 'N/A'}." + ) + return summary + except Exception: + return "No summary available" + + +def generate_insights(aggregates: dict): + """Generate human-readable insights and recommendations. + + If OpenAI API key is present, attempt a short chat completion. If not, + return a simple deterministic summary and a few heuristic recommendations. + """ + try: + prompt_summary = _summarize_aggregates(aggregates) + if OPENAI_API_KEY: + try: + import openai + openai.api_key = OPENAI_API_KEY + system = "You are an analytics assistant for a collaborative drawing app. Provide a short summary and 3 actionable recommendations to improve collaboration and room health." + response = openai.ChatCompletion.create( + model="gpt-4o-mini", + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": f"Here are aggregates: {json.dumps(aggregates)}. Produce a short summary and 3 actionable recommendations."} + ], + max_tokens=400, + temperature=0.6, + ) + text = response.choices[0].message.content + return {"summary": text, "source": "openai"} + except Exception: + logger.exception('OpenAI call failed, falling back to heuristic summary') + + # Fallback heuristic + summary = _summarize_aggregates(aggregates) + recommendations = [] + if aggregates.get('active_users', 0) < 2: + recommendations.append('Encourage users to invite collaborators or schedule group drawing sessions to improve engagement.') + if aggregates.get('avg_stroke_rate', 0) < 1: + recommendations.append('Introduce prompts or templates to encourage drawing activity.') + if aggregates.get('anomaly_score', 0) > 0.7: + recommendations.append('Investigate sudden surges in activity — could be bot behavior or an event-driven spike.') + if not recommendations: + recommendations = [ + 'Highlight active users in the room to promote collaboration.', + 'Surface weekly summary emails with top contributors and trending palettes.' + ] + return {"summary": summary, "recommendations": recommendations, "source": "heuristic"} + except Exception: + logger.exception('generate_insights error') + return {"summary": "Unable to generate insights.", "recommendations": []} diff --git a/backend/tests/test_analytics_service.py b/backend/tests/test_analytics_service.py new file mode 100644 index 00000000..eb2aea57 --- /dev/null +++ b/backend/tests/test_analytics_service.py @@ -0,0 +1,53 @@ +import sys +import types +import time + + +class FakeColl: + def __init__(self): + self.storage = [] + + def insert_one(self, doc): + # emulate pymongo returning an InsertOneResult-like object + self.storage.append(dict(doc)) + class R: pass + r = R() + r.inserted_id = len(self.storage) - 1 + return r + + def find(self, q=None): + # return all docs matching roomId if provided + q = q or {} + room = q.get('roomId') + if room: + return [d for d in self.storage if d.get('roomId') == room] + return list(self.storage) + + +def test_ingest_and_query_recent(monkeypatch): + # Inject a fake services.db module to avoid importing real dependencies (redis/mongo) + fake_db = types.ModuleType('services.db') + fake_analytics = FakeColl() + fake_db.analytics_coll = fake_analytics + sys.modules['services.db'] = fake_db + + # Now import the analytics service (it will import services.db from sys.modules) + import services.analytics_service as analytics_service + + ev = analytics_service.ingest_event({ + 'roomId': 'room-test-1', + 'userId': 'user-123', + 'eventType': 'stroke_created', + 'payload': {'color': '#ff0000'}, + 'ts': int(time.time() * 1000) + }) + + assert ev is not None + # userId should be removed and anonUserId present + assert 'anonUserId' in ev and 'userId' not in ev + + recent = analytics_service.query_recent('room-test-1', limit=10) + assert isinstance(recent, list) + assert len(recent) >= 1 + # Ensure stored doc contains our eventType + assert any(r.get('eventType') == 'stroke_created' for r in recent) diff --git a/backend/tests/test_insights_generator.py b/backend/tests/test_insights_generator.py new file mode 100644 index 00000000..6a2df574 --- /dev/null +++ b/backend/tests/test_insights_generator.py @@ -0,0 +1,31 @@ +import importlib.util +import os + + +def load_insights_module(): + path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'backend', 'services', 'insights_generator.py')) + spec = importlib.util.spec_from_file_location('insights_generator', path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def test_generate_insights_heuristic(): + ig = load_insights_module() + # Force heuristic path by ensuring OPENAI API key is unset on the module + setattr(ig, 'OPENAI_API_KEY', None) + + aggregates = { + 'total_strokes': 42, + 'active_users': 1, + 'top_colors': ['#ff0000', '#00ff00'], + 'collaboration_pairs': [] , + 'avg_stroke_rate': 0.2, + 'anomaly_score': 0.1 + } + + out = ig.generate_insights(aggregates) + assert isinstance(out, dict) + assert 'summary' in out + # recommendations should be present when activity is low + assert 'recommendations' in out and isinstance(out['recommendations'], list) diff --git a/backend/workers/analytics_aggregation_worker.py b/backend/workers/analytics_aggregation_worker.py new file mode 100644 index 00000000..d3fe2b39 --- /dev/null +++ b/backend/workers/analytics_aggregation_worker.py @@ -0,0 +1,78 @@ +""" +Simple aggregation worker for analytics events. +This script can be invoked periodically or run as a long-lived background process +to summarize raw events into per-room aggregates. +""" +import time +import logging +from services.db import analytics_coll, analytics_aggregates_coll +from pymongo import UpdateOne +from collections import Counter, defaultdict + +logger = logging.getLogger(__name__) + + +def aggregate_once(batch_limit=1000): + # Pull recent unprocessed events. For simplicity we process events by ts window. + docs = list(analytics_coll.find({}).sort([('ts', -1)]).limit(batch_limit)) + if not docs: + return 0 + + rooms = defaultdict(list) + for d in docs: + rooms[d.get('roomId')].append(d) + + ops = [] + for room, events in rooms.items(): + total_strokes = sum(1 for e in events if e.get('eventType') == 'stroke_created') + active_users = len({e.get('anonUserId') for e in events if e.get('anonUserId')}) + color_counter = Counter() + pair_counter = Counter() + timestamps = [e.get('ts') for e in events if e.get('ts')] + for e in events: + payload = e.get('payload') or {} + if payload.get('color'): + color_counter[payload.get('color')] += 1 + # collaboration pairs (simple heuristic in payload) + if payload.get('from') and payload.get('to'): + pair = (payload.get('from'), payload.get('to')) + pair_counter[pair] += 1 + + agg = { + 'roomId': room, + 'total_strokes': total_strokes, + 'active_users': active_users, + 'top_colors': [c for c, _ in color_counter.most_common(10)], + 'collaboration_pairs': [ [a,b,count] for (a,b),count in pair_counter.most_common(10) ], + 'first_ts': min(timestamps) if timestamps else None, + 'last_ts': max(timestamps) if timestamps else None, + 'avg_stroke_rate': (total_strokes / ((max(timestamps)-min(timestamps))/1000)) if total_strokes and len(timestamps) > 1 and max(timestamps) != min(timestamps) else total_strokes, + } + + # basic anomaly score: high stroke rate or many distinct anon users + agg['anomaly_score'] = float(min(1.0, (agg['avg_stroke_rate'] / 100.0) + (agg['active_users'] / 100.0))) + + ops.append(UpdateOne({'roomId': room}, {'$set': agg}, upsert=True)) + + if ops: + analytics_aggregates_coll.bulk_write(ops) + return len(docs) + + +def run_loop(sleep_secs=10): + logger.info('Starting analytics aggregation worker loop') + try: + while True: + n = aggregate_once() + if n == 0: + time.sleep(sleep_secs) + else: + # brief pause to avoid tight loops + time.sleep(1) + except KeyboardInterrupt: + logger.info('Worker stopped') + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + run_loop() diff --git a/frontend/package.json b/frontend/package.json index b963a141..36abfd82 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -2,7 +2,7 @@ "name": "canvas-app", "version": "0.1.0", "private": true, - "dependencies": { + "dependencies": {"d3": "^7.10.0", "@emotion/react": "^11.13.5", "@emotion/styled": "^11.13.5", "@mui/icons-material": "^6.4.7", @@ -64,7 +64,7 @@ "react": "^17.0.0 || ^18.0.0", "react-dom": "^17.0.0 || ^18.0.0" }, - "devDependencies": { + "devDependencies": {"d3": "^7.10.0", "@playwright/test": "^1.55.1", "playwright": "^1.36.0" } diff --git a/frontend/src/components/Analytics/CollaborationGraph.jsx b/frontend/src/components/Analytics/CollaborationGraph.jsx new file mode 100644 index 00000000..838f86a7 --- /dev/null +++ b/frontend/src/components/Analytics/CollaborationGraph.jsx @@ -0,0 +1,110 @@ +import React, { useRef, useEffect } from 'react'; +import * as d3 from 'd3'; + +// Force-directed collaboration graph using d3-force. +export default function CollaborationGraph({ pairs = [] , width = 600, height = 300}) { + const ref = useRef(); + + useEffect(() => { + const nodeMap = new Map(); + const links = []; + pairs.forEach((p) => { + const a = p[0]; + const b = p[1]; + const w = p[2] || 1; + if (!nodeMap.has(a)) nodeMap.set(a, { id: a }); + if (!nodeMap.has(b)) nodeMap.set(b, { id: b }); + links.push({ source: a, target: b, weight: w }); + }); + const nodes = Array.from(nodeMap.values()); + + const svg = d3.select(ref.current); + svg.selectAll('*').remove(); + + svg.attr('viewBox', `0 0 ${width} ${height}`).style('width', '100%').style('height', 'auto'); + + const link = svg.append('g') + .attr('stroke', '#999') + .attr('stroke-opacity', 0.6) + .selectAll('line') + .data(links) + .join('line') + .attr('stroke-width', d => Math.sqrt(d.weight)); + + const node = svg.append('g') + .attr('stroke', '#fff') + .attr('stroke-width', 1.5) + .selectAll('circle') + .data(nodes) + .join('circle') + .attr('r', 8) + .attr('fill', '#25D8C5') + .call(drag(simulation)); + + const label = svg.append('g') + .selectAll('text') + .data(nodes) + .join('text') + .attr('font-size', 10) + .attr('dx', 12) + .attr('dy', '.35em') + .text(d => d.id); + + function ticked() { + link + .attr('x1', d => d.source.x) + .attr('y1', d => d.source.y) + .attr('x2', d => d.target.x) + .attr('y2', d => d.target.y); + + node + .attr('cx', d => d.x) + .attr('cy', d => d.y); + + label + .attr('x', d => d.x) + .attr('y', d => d.y); + } + + const simulation = d3.forceSimulation(nodes) + .force('link', d3.forceLink(links).id(d => d.id).distance(50).strength(d => Math.min(0.9, d.weight / 10))) + .force('charge', d3.forceManyBody().strength(-120)) + .force('center', d3.forceCenter(width / 2, height / 2)) + .on('tick', ticked); + + function drag(sim) { + function dragstarted(event, d) { + if (!event.active) sim.alphaTarget(0.3).restart(); + d.fx = d.x; + d.fy = d.y; + } + + function dragged(event, d) { + d.fx = event.x; + d.fy = event.y; + } + + function dragended(event, d) { + if (!event.active) sim.alphaTarget(0); + d.fx = null; + d.fy = null; + } + + return d3.drag() + .on('start', dragstarted) + .on('drag', dragged) + .on('end', dragended); + } + + return () => { + simulation?.stop(); + }; + }, [pairs, width, height]); + + return ( +