feat: export with spreadsheet

Author: 7086cmd

routers/activities_v2_router.py

@@ -410,11 +410,17 @@ async def modify_activity_status_v2(
             detail="Activity status can only be modified from pending to effective or refused",
         )
 
-    await volunteer.validate_check_permission(user, target_activity)
+    await volunteer.validate_check_permission(user, target_activity, strict=True)
 
     result = await db.zvms_new.get_collection("activities").update_one(
         {"_id": validate_object_id(activity_id)},
-        {"$set": {"status": status.status}},
+        {
+            "$set": {
+                "status": status.status,
+                "approver": str(user["id"]),
+                "updatedAt": datetime.now(),
+            }
+        },
     )
 
     if result.modified_count == 0:

routers/exports_router.py

@@ -4,14 +4,24 @@
 from fastapi import APIRouter, HTTPException, Depends, Request
 import tempfile
 
-from config import BASE_ON_CAMPUS, ON_TO_OFF_RATE, MAX_EXCEED_DISCOUNT, BASE_OFF_CAMPUS, OFF_TO_ON_RATE
+from config import (
+    BASE_ON_CAMPUS,
+    ON_TO_OFF_RATE,
+    MAX_EXCEED_DISCOUNT,
+    BASE_OFF_CAMPUS,
+    OFF_TO_ON_RATE,
+)
 from typings.export import ExportFormat, ExportTask, ExportStatus, ExportVariant
 from util.calculate import calculate_user_time
 from fastapi.responses import FileResponse
 from io import BytesIO
 import base64
 import json
-from util.object_id import get_current_user, validate_object_id, compulsory_temporary_token
+from util.object_id import (
+    get_current_user,
+    validate_object_id,
+    compulsory_temporary_token,
+)
 from datetime import datetime
 from database import db
 import pandas as pd
@@ -26,6 +36,7 @@ class CreateExport(BaseModel):
     allow_cache: bool = False
     include_description: bool = False
 
+
 tokens = []
 
 
@@ -54,14 +65,22 @@ async def process_task(task_id: str):
                 )
             else:
                 user_time = await calculate_user_time(
-                    str(user["_id"]), allow_cache=task["allow_cache"],
+                    str(user["_id"]),
+                    allow_cache=task["allow_cache"],
                     attach_description=include_description,
                 )
             more_on_campus = min(
-                round(max(user_time["off-campus"] - BASE_OFF_CAMPUS, 1) * OFF_TO_ON_RATE, 0), MAX_EXCEED_DISCOUNT
+                round(
+                    max(user_time["off-campus"] - BASE_OFF_CAMPUS, 1) * OFF_TO_ON_RATE,
+                    0,
+                ),
+                MAX_EXCEED_DISCOUNT,
             )
             more_off_campus = min(
-                round(max(user_time["on-campus"] - BASE_ON_CAMPUS, 1) * ON_TO_OFF_RATE, 0), MAX_EXCEED_DISCOUNT
+                round(
+                    max(user_time["on-campus"] - BASE_ON_CAMPUS, 1) * ON_TO_OFF_RATE, 0
+                ),
+                MAX_EXCEED_DISCOUNT,
             )
             user_time["on-campus"] += more_on_campus
             user_time["off-campus"] += more_off_campus
@@ -83,7 +102,9 @@ async def process_task(task_id: str):
                 "On Campus": user_time["on-campus"],
                 "Off Campus": user_time["off-campus"],
                 "Social Practice": user_time["social-practice"],
-                "Description": "" if not include_description else user_time.get("description", ""),
+                "Description": ""
+                if not include_description
+                else user_time.get("description", ""),
             }
             result.append(doc)
             task["percentage"] = (idx + 1) / len(users) * 100
@@ -240,6 +261,7 @@ async def get_export(task_id: str):
     task["_id"] = str(task["_id"])
     return {"code": 200, "status": "ok", "data": task}
 
+
 @router.post("/reports")
 async def request_download_reports(
     user=Depends(compulsory_temporary_token),
@@ -248,14 +270,15 @@ async def request_download_reports(
         raise HTTPException(status_code=403, detail="Permission denied")
     global tokens
     token_data = {
-        'token': uuid.uuid4().hex,
-        'granted_to': user['id'],
-        'expires_at': datetime.now().timestamp() + 300  # 5 minutes
+        "token": uuid.uuid4().hex,
+        "granted_to": user["id"],
+        "expires_at": datetime.now().timestamp() + 300,  # 5 minutes
     }
     token_data_encoded = base64.b64encode(json.dumps(token_data).encode()).decode()
     tokens.append(token_data)
     return token_data_encoded
 
+
 @router.get("/reports/download")
 async def download_reports(
     request: Request,
@@ -267,14 +290,14 @@ async def download_reports(
     except Exception:
         raise HTTPException(status_code=400, detail="Invalid token format")
 
-    if not any(t['token'] == token_data['token'] for t in tokens):
+    if not any(t["token"] == token_data["token"] for t in tokens):
         raise HTTPException(status_code=403, detail="Invalid or expired token")
 
-    if datetime.now().timestamp() > token_data['expires_at']:
+    if datetime.now().timestamp() > token_data["expires_at"]:
         raise HTTPException(status_code=403, detail="Token expired")
 
     # Remove the token after use
-    tokens = [t for t in tokens if t['token'] != token_data['token']]
+    tokens = [t for t in tokens if t["token"] != token_data["token"]]
 
     # response to `export.tar.gz`, which is a tar.gz file containing all reports and already exists in the server
     file_path = "./data/export.tar.gz"
@@ -284,9 +307,7 @@ async def download_reports(
         file_path,
         media_type="application/gzip",
         filename="export.tar.gz",
-        headers={
-            "Content-Disposition": "attachment; filename=export.tar.gz"
-        }
+        headers={"Content-Disposition": "attachment; filename=export.tar.gz"},
     )
 
 
@@ -303,16 +324,19 @@ async def get_export_file(task_id: str, language: str = "en"):
     ) as tmp:
         result = pd.DataFrame(task["result"]).sort_values("_id")
         if language == "zh-CN":
-            result.rename(columns={
-                "_id": "数据库 ID",
-                "Name": "姓名",
-                "ID": "学号",
-                "Group": "班级",
-                "On Campus": "校内义工时长",
-                "Off Campus": "校外义工时长",
-                "Social Practice": "社会实践时长",
-                "Description": "描述",
-            }, inplace=True)
+            result.rename(
+                columns={
+                    "_id": "数据库 ID",
+                    "Name": "姓名",
+                    "ID": "学号",
+                    "Group": "班级",
+                    "On Campus": "校内义工时长",
+                    "Off Campus": "校外义工时长",
+                    "Social Practice": "社会实践时长",
+                    "Description": "描述",
+                },
+                inplace=True,
+            )
         if task["format"] == "excel":
             result.to_excel(tmp.name, index=False)
         elif task["format"] == "csv":

routers/times_router.py

@@ -5,7 +5,8 @@
 from util.calculate import calculate_user_time
 from util.object_id import (
     optional_current_user,
-    validate_object_id, get_current_user,
+    validate_object_id,
+    get_current_user,
 )
 from database import db
 
@@ -19,7 +20,7 @@ async def read_times(
     perpage: int = 5,
     sort: str = "id",
     asc: bool = True,
-    user = Depends(get_current_user),
+    user=Depends(get_current_user),
 ):
     """
     Query users

routers/users_v2_router.py

@@ -179,10 +179,14 @@ async def get_user_time_statistics_v2(
         "social-practice": {
             "value": user_time["social-practice"],
             "group": find_percentile_threshold(
-                group_indicators["social-practice"], user_time["social-practice"], "social-practice"
+                group_indicators["social-practice"],
+                user_time["social-practice"],
+                "social-practice",
             ),
             "grade": find_percentile_threshold(
-                grade_indicators["social-practice"], user_time["social-practice"], "social-practice"
+                grade_indicators["social-practice"],
+                user_time["social-practice"],
+                "social-practice",
             ),
         },
     }

tasks/compute_time.py

@@ -15,7 +15,8 @@
 )
 from database import db
 from util.calculate import calculate_user_time
-from util.statement import create_statement_from_kernel_data
+from util.statement import create_statement_from_kernel_data, TRANSLATIONS
+from utils import validate_object_id
 
 
 async def compute_time():
@@ -35,51 +36,111 @@ async def compute_ay_time():
     users = await db.zvms.get_collection("users").find({}).to_list(None)
     now = datetime.now()
     ay = now.year if now.month >= 9 else now.year - 1
-    soy = datetime.now().replace(month=8, day=1, year=ay, hour=0, minute=0, second=0, microsecond=0)
-    eoy = datetime.now().replace(month=7, day=31, year=ay + 1, hour=23, minute=59, second=59, microsecond=999999)
+    soy = datetime.now().replace(
+        month=8, day=1, year=ay, hour=0, minute=0, second=0, microsecond=0
+    )
+    eoy = datetime.now().replace(
+        month=7, day=31, year=ay + 1, hour=23, minute=59, second=59, microsecond=999999
+    )
     for user in users:
         user_id = str(user["_id"])
         entry = user_id[:4]
-        result = await calculate_user_time(user_id, date_start=soy, date_end=eoy, allow_cache=False)
-        await db.zvms_new.get_collection("time_academic_year").delete_many({"user": user_id})
-        await db.zvms_new.get_collection("time_academic_year").insert_one({
-            "user": user_id,
-            "entry": entry,
-            "academic_year": ay,
-            "start_of_year": soy,
-            "end_of_year": eoy,
-            **result
-        })
+        result = await calculate_user_time(
+            user_id, date_start=soy, date_end=eoy, allow_cache=False
+        )
+        await db.zvms_new.get_collection("time_academic_year").delete_many(
+            {"user": user_id}
+        )
+        await db.zvms_new.get_collection("time_academic_year").insert_one(
+            {
+                "user": user_id,
+                "entry": entry,
+                "academic_year": ay,
+                "start_of_year": soy,
+                "end_of_year": eoy,
+                **result,
+            }
+        )
+
 
 async def generate_reports():
-    classes = await db.zvms.get_collection("groups").find({"type": "class"}).to_list(None)
-    if not os.path.exists('./export'):
-        os.makedirs('./export')
+    classes = (
+        await db.zvms.get_collection("groups").find({"type": "class"}).to_list(None)
+    )
+    await db.zvms_new.get_collection("daily_exports").delete_many({})
+    db_data = await db.zvms_new.get_collection("daily_exports").insert_one(
+        {"created_at": datetime.now(), "status": "processing", "data": []}
+    )
+    db_id = validate_object_id(db_data.inserted_id)
+    if not os.path.exists("./export"):
+        os.makedirs("./export")
     else:
-        for item in os.listdir('./export'):
-            item_path = os.path.join('./export', item)
+        for item in os.listdir("./export"):
+            item_path = os.path.join("./export", item)
             if os.path.isdir(item_path):
                 for file in os.listdir(item_path):
                     file_path = os.path.join(item_path, file)
-                    if file.endswith('.pdf'):
+                    if file.endswith(".pdf"):
                         os.remove(file_path)
                 os.rmdir(item_path)
     for class_id in classes:
-        users = await db.zvms.get_collection("users").find({"group": str(class_id["_id"])}).to_list(None)
+        users = (
+            await db.zvms.get_collection("users")
+            .find({"group": str(class_id["_id"])})
+            .to_list(None)
+        )
         os.makedirs(f'./export/{class_id["name"]}', exist_ok=True)
         for user in users:
             user_id = str(user["id"])
             entry = int(user_id[:4])
-            soy = datetime.now().replace(month=8, day=1, year=entry, hour=0, minute=0, second=0, microsecond=0)
-            eoy = datetime.now().replace(month=7, day=31, year=entry + 3, hour=23, minute=59, second=59, microsecond=999999)
-            await create_statement_from_kernel_data(str(user['_id']), (soy, eoy), filename=f'./export/{class_id['name']}/{user['id']}_{user["name"]}.pdf', language='zh')
-            print(f"Exported statement for {user['name']} in class {class_id['name']}.")
+            soy = datetime.now().replace(
+                month=8, day=1, year=entry, hour=0, minute=0, second=0, microsecond=0
+            )
+            eoy = datetime.now().replace(
+                month=7,
+                day=31,
+                year=entry + 3,
+                hour=23,
+                minute=59,
+                second=59,
+                microsecond=999999,
+            )
+            filename, spreadsheet = await create_statement_from_kernel_data(
+                str(user["_id"]),
+                (soy, eoy),
+                filename=f'./export/{class_id['name']}/{user['id']}_{user["name"]}.pdf',
+                language="zh",
+            )
+            await db.zvms_new.get_collection("daily_exports").update_one(
+                {"_id": db_id},
+                {
+                    "$push": {
+                        "data": {
+                            **spreadsheet,
+                            "filename": filename,
+                        }
+                    }
+                },
+            )
+            print(
+                f"Exported statement for {user['name']} in class {class_id['name']} (saved to {filename})."
+            )
         print(f"Exported statements for class {class_id['name']}.")
     # Create a tar.gz archive of the export directory
-    if os.path.exists('./data/export.tar.gz'):
-        os.remove('./data/export.tar.gz')
-    with tarfile.open('./data/export.tar.gz', 'w:gz') as tar:
-        tar.add('./export', arcname='export')
+    spreadsheet_items = await db.zvms_new.get_collection("daily_exports").find_one(
+        {"_id": db_id}
+    )
+    spreadsheet_items = spreadsheet_items.get("data", [])
+    df = pd.DataFrame(spreadsheet_items)
+    df.rename(columns=TRANSLATIONS["zh"]["spreadsheet"], inplace=True)
+    if os.path.exists("./export/summary.xlsx"):
+        os.remove("./export/summary.xlsx")
+    df.to_excel(f"./export/summary.xlsx", index=False)
+    if os.path.exists("./data/export.tar.gz"):
+        os.remove("./data/export.tar.gz")
+    with tarfile.open("./data/export.tar.gz", "w:gz") as tar:
+        tar.add("./export", arcname="export")
+    return "./data/export.tar.gz"
 
 
 def describe_percentile(items: np.ndarray, step: int, bound: float) -> dict[str, float]:
@@ -142,7 +203,14 @@ def process_time_data(time_df: pd.DataFrame) -> pd.DataFrame:
         1,
     )
     time_df["social-practice"] = time_df["social_practice"]
-    time_df["diff"] = BASE_ON_CAMPUS + BASE_OFF_CAMPUS + BASE_SOCIAL_PRACTICE - time_df['on-campus'] - time_df['off-campus'] - time_df['social-practice']
+    time_df["diff"] = (
+        BASE_ON_CAMPUS
+        + BASE_OFF_CAMPUS
+        + BASE_SOCIAL_PRACTICE
+        - time_df["on-campus"]
+        - time_df["off-campus"]
+        - time_df["social-practice"]
+    )
     return time_df