feat: first version

Cygra · Jan 11, 2025 · 43a6fa5 · 43a6fa5
1 parent 3c298bc
commit 43a6fa5
Show file tree

Hide file tree

Showing 8 changed files with 1,169 additions and 146 deletions.
diff --git a/README.md b/README.md
@@ -2,6 +2,14 @@
 
 ![Next JS](https://img.shields.io/badge/Next-black?style=for-the-badge&logo=next.js&logoColor=white)
 
+基于 Next.js 和 [Mediapipe](https://ai.google.dev/edge/mediapipe/solutions/guide) 实现的模拟 Bilibili 实时防挡脸弹幕效果。
+
+A simulation of Bilibili's real-time face-masked danmaku barrage effect based on Next.js and Mediapipe.
+
+| Bilibili                          | Danmaku Mask                   |
+| --------------------------------- | ------------------------------ |
+| ![bilibili](./docs/bilibili.avif) | ![example](./docs/example.JPG) |
+
 ## Getting Started
 
 ```sh

diff --git a/docs/bilibili.avif b/docs/bilibili.avif
diff --git a/docs/example.JPG b/docs/example.JPG
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -10,6 +10,13 @@
   },
   "dependencies": {
     "@mediapipe/tasks-vision": "^0.10.20",
+    "@next/third-parties": "^15.1.4",
+    "@nextui-org/button": "^2.2.9",
+    "@nextui-org/form": "^2.1.8",
+    "@nextui-org/input": "^2.4.8",
+    "@nextui-org/system": "^2.4.6",
+    "@nextui-org/theme": "^2.4.5",
+    "framer-motion": "^11.17.0",
     "next": "15.1.4",
     "react": "^19.0.0",
     "react-dom": "^19.0.0"

diff --git a/src/app/layout.tsx b/src/app/layout.tsx
@@ -1,6 +1,8 @@
 import type { Metadata } from "next";
 import { Geist, Geist_Mono } from "next/font/google";
 import "./globals.css";
+import Head from "next/head";
+import { GoogleAnalytics } from "@next/third-parties/google";
 
 const geistSans = Geist({
   variable: "--font-geist-sans",
@@ -24,6 +26,35 @@ export default function RootLayout({
 }>) {
   return (
     <html lang="en">
+      <GoogleAnalytics gaId="G-2ZP5YG0JDG" />
+      <Head>
+        <title>Danmaku Mask</title>
+        <meta
+          name="description"
+          content={
+            "基于 Next.js 和 Mediapipe tasks-vision Image Segmenter 实现的模拟 Bilibili 实时防挡脸弹幕效果。" +
+            "使用机器学习在人脸上渲染一个实时的 mask。" +
+            "This site is developed with Next.js and Mediapipe tasks-vision Image Segmenter." +
+            " Implmenting machine learning to detect human face and render a real-time mask."
+          }
+        />
+        <meta property="og:title" content="Danmaku Mask" />
+        <meta property="og:site_name" content="Danmaku Mask" />
+        <meta property="og:type" content="website" />
+        <meta
+          property="og:description"
+          content={
+            "基于 Next.js 和 Mediapipe tasks-vision Image Segmenter 实现的模拟 Bilibili 实时防挡脸弹幕效果。" +
+            "使用机器学习在人脸上渲染一个实时的 mask。" +
+            "This site is developed with Next.js and Mediapipe tasks-vision Image Segmenter." +
+            " Implmenting machine learning to detect human face and render a real-time mask."
+          }
+        />
+        <meta
+          property="og:url"
+          content="https://cygra.github.io/danmaku-mask/"
+        />
+      </Head>
       <body
         className={`${geistSans.variable} ${geistMono.variable} antialiased`}
       >

diff --git a/src/app/page.tsx b/src/app/page.tsx
@@ -1,29 +1,68 @@
 "use client";
 
-import { useEffect, useRef } from "react";
+import { useEffect, useRef, useState } from "react";
 
 import {
   ImageSegmenter,
   FilesetResolver,
   ImageSegmenterResult,
 } from "@mediapipe/tasks-vision";
+import { Input } from "@nextui-org/input";
+import { Form } from "@nextui-org/form";
+import { Button } from "@nextui-org/button";
 
-const legendColors = [
-  [147, 170, 0, 255],
-  [166, 189, 215, 255],
+const LOREM = [
+  "Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
+  "Curabitur luctus massa quis nunc luctus aliquam.",
+  "Maecenas eget nisi id neque vehicula venenatis in rutrum dui.",
+  "Morbi vulputate lectus vel porta scelerisque.",
+  "Etiam ornare nibh ac sapien laoreet varius.",
+  "In et enim quis arcu venenatis tempus ac a justo.",
+  "Quisque in ipsum placerat, auctor justo ac, cursus ipsum.",
+  "Integer et purus pharetra, efficitur libero ut, finibus libero.",
+  "Nulla venenatis velit et eros congue sollicitudin.",
+  "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit...",
+  "超级无敌炫酷吊炸天",
 ];
 
 export default function Home() {
-  const process = async () => {
+  const [canvasSize, setCanvasSize] = useState([0, 0]);
+  const [danmakuMap, setDanmakuMap] = useState<
+    Map<number, Map<string, string>>
+  >(() => {
+    const m = new Map();
+    m.set(0, new Map());
+    m.set(1, new Map());
+    m.set(2, new Map());
+    m.set(3, new Map());
+    m.set(4, new Map());
+    m.set(5, new Map());
+    m.set(6, new Map());
+    return m;
+  });
+  const [intervalId, setIntervalId] = useState<NodeJS.Timeout>();
+
+  const prepareVideoStream = async () => {
     const stream = await navigator.mediaDevices.getUserMedia({
       audio: false,
       video: true,
     });
 
     if (videoRef.current) {
       videoRef.current.srcObject = stream;
+      videoRef.current.addEventListener("loadeddata", () => {
+        if (
+          videoRef.current?.readyState &&
+          videoRef.current?.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA
+        ) {
+          const { videoWidth, videoHeight } = videoRef.current;
+          setCanvasSize([videoWidth, videoHeight]);
+        }
+      });
     }
+  };
 
+  const process = async (videoWidth: number, videoHeight: number) => {
     let lastWebcamTime = -1;
     const vision = await FilesetResolver.forVisionTasks(
       "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
@@ -40,56 +79,47 @@ export default function Home() {
       runningMode: "VIDEO",
     });
 
-    const canvas = canvasRef.current;
+    const canvas = maskCanvasRef.current;
     const ctx = canvas?.getContext("2d");
-    const video = videoRef.current;
+    const video = videoRef.current!;
 
     const callbackForVideo = (result: ImageSegmenterResult) => {
       const imageData = ctx?.getImageData(
         0,
         0,
-        video!.videoWidth || 640,
-        video!.videoHeight || 480
+        videoWidth || 640,
+        videoHeight || 480
       ).data;
       const mask = result?.categoryMask?.getAsFloat32Array();
 
       let j = 0;
       if (mask && imageData) {
         for (let i = 0; i < mask.length; ++i) {
-          const maskVal = Math.round(mask[i] * 255.0);
-          const legendColor = legendColors[maskVal % legendColors.length];
-          imageData[j] = legendColor[0] + imageData[j];
-          imageData[j + 1] = (legendColor[1] + imageData[j + 1]) / 2;
-          imageData[j + 2] = (legendColor[2] + imageData[j + 2]) / 2;
-          imageData[j + 3] = (legendColor[3] + imageData[j + 3]) / 2;
+          if (mask[i] === 0) {
+            imageData[j + 3] = 0;
+          }
           j += 4;
         }
         const uint8Array = new Uint8ClampedArray(imageData.buffer);
         const dataNew = new ImageData(
           uint8Array,
-          video!.videoWidth || 640,
-          video!.videoHeight || 480
+          videoWidth || 640,
+          videoHeight || 480
         );
         ctx?.putImageData(dataNew, 0, 0);
         requestAnimationFrame(predictWebcam);
       }
     };
 
     const predictWebcam = async () => {
-      if (!video || !canvasRef.current) return;
+      if (!video || !maskCanvasRef.current) return;
       if (video.currentTime === lastWebcamTime) {
         requestAnimationFrame(predictWebcam);
         return;
       }
 
       lastWebcamTime = video.currentTime;
-      ctx?.drawImage(
-        video,
-        0,
-        0,
-        video!.videoWidth || 640,
-        video!.videoHeight || 480
-      );
+      ctx?.drawImage(video, 0, 0, videoWidth || 640, videoHeight || 480);
       if (imageSegmenter === undefined) {
         return;
       }
@@ -100,18 +130,131 @@ export default function Home() {
     predictWebcam();
   };
 
+  const renderDanmaku = (danmaku: string) => {
+    const row = Math.floor(Math.random() * 7);
+    const timeStamp = new Date().getTime().toString();
+
+    setDanmakuMap((t) =>
+      new Map(t).set(row, new Map(t.get(row)).set(timeStamp, danmaku))
+    );
+
+    setTimeout(() => {
+      setDanmakuMap((t) => {
+        const targetMap = new Map(t.get(row));
+        targetMap.delete(timeStamp);
+        return new Map(t).set(row, targetMap);
+      });
+    }, 6000);
+  };
+
+  const handleAutoGenerate = () => {
+    if (intervalId) {
+      clearInterval(intervalId);
+      setIntervalId(undefined);
+    } else {
+      const id = setInterval(() => {
+        const idx = Math.floor(Math.random() * LOREM.length);
+        renderDanmaku(LOREM[idx]);
+      }, 500);
+      renderDanmaku("超级无敌炫酷吊炸天");
+      setIntervalId(id);
+    }
+  };
+
   useEffect(() => {
-    process();
+    prepareVideoStream();
   }, []);
 
+  useEffect(() => {
+    if (canvasSize[0] && canvasSize[1]) {
+      process(canvasSize[0], canvasSize[1]);
+    }
+  }, [canvasSize]);
+
   const videoRef = useRef<HTMLVideoElement>(null);
-  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const maskCanvasRef = useRef<HTMLCanvasElement>(null);
+  const formRef = useRef<HTMLFormElement>(null);
 
   return (
-    <div className="flex items-center min-h-screen p-8 w-full justify-center">
-      <canvas ref={canvasRef} width={640} height={480} className="w-[800px]" />
+    <div className="flex flex-col items-center min-h-screen p-8 w-full justify-center">
+      <div
+        className="rounded-lg overflow-hidden relative w-[800px]"
+        style={{
+          height:
+            canvasSize[0] && canvasSize[1]
+              ? (800 * canvasSize[1]) / canvasSize[0]
+              : 0,
+        }}
+      >
+        <video
+          playsInline
+          ref={videoRef}
+          autoPlay
+          muted
+          className={"absolute w-[800px]"}
+          style={{
+            height:
+              canvasSize[0] && canvasSize[1]
+                ? (800 * canvasSize[1]) / canvasSize[0]
+                : 0,
+            transform: "rotateY(180deg)",
+          }}
+        />
+
+        {Array.from(danmakuMap.entries()).map((it) =>
+          Array.from(it[1].entries()).map((d) => (
+            <div
+              key={it[1] + d[0] + d[1]}
+              className={`absolute pt-1 animate-danmaku text-4xl font-semibold whitespace-nowrap`}
+              style={{
+                textShadow: "#000 1px 0 1px",
+                animationFillMode: "forwards",
+                top: 40 * it[0],
+              }}
+            >
+              {d[1]}
+            </div>
+          ))
+        )}
 
-      <video playsInline ref={videoRef} autoPlay className={"hidden"}></video>
+        <canvas
+          ref={maskCanvasRef}
+          width={canvasSize[0]}
+          height={canvasSize[1]}
+          className="absolute w-[800px]"
+          style={{ transform: "rotateY(180deg)" }}
+        />
+      </div>
+      {canvasSize[0] ? (
+        <Form
+          ref={formRef}
+          className="w-full max-w-xl mt-2 flex flex-row"
+          validationBehavior="native"
+          onSubmit={(e) => {
+            e.preventDefault();
+            const data = Object.fromEntries(new FormData(e.currentTarget));
+            const { danmaku } = data;
+            renderDanmaku(danmaku.toString());
+            formRef.current?.reset();
+          }}
+        >
+          <Input
+            isRequired
+            disabled={!!intervalId}
+            errorMessage="Please enter somthing"
+            name="danmaku"
+            placeholder="Write something, click [Enter] to send"
+            type="text"
+          />
+          <Button
+            className="ml-2"
+            color={intervalId ? "danger" : "primary"}
+            onPress={handleAutoGenerate}
+          >
+            {intervalId ? "Stop" : "Auto Generate"}
+          </Button>
+        </Form>
+      ) : null}
     </div>
   );
 }
diff --git a/tailwind.config.ts b/tailwind.config.ts
@@ -1,18 +1,35 @@
+import { nextui } from "@nextui-org/theme";
 import type { Config } from "tailwindcss";
 
 export default {
   content: [
     "./src/pages/**/*.{js,ts,jsx,tsx,mdx}",
     "./src/components/**/*.{js,ts,jsx,tsx,mdx}",
     "./src/app/**/*.{js,ts,jsx,tsx,mdx}",
+    "./node_modules/@nextui-org/theme/dist/components/(button|form|input|ripple|spinner).js"
   ],
   theme: {
     extend: {
       colors: {
         background: "var(--background)",
         foreground: "var(--foreground)",
       },
+      animation: {
+        danmaku: "danmaku 6s linear",
+      },
+      keyframes: {
+        danmaku: {
+          "0%": {
+            transform: "translateX(0)",
+            left: "100%",
+          },
+          "100%": {
+            transform: "translateX(-100%)",
+            left: "0px",
+          },
+        },
+      },
     },
   },
-  plugins: [],
+  plugins: [nextui()],
 } satisfies Config;