ml5/demos/09-neural-network-rock-paper-scissors.html

<!DOCTYPE html>
<html lang="en" class="loading">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>Neural Network Rock Paper Scissors</title>
  <link rel="stylesheet" href="css/normalize.css">
  <link rel="stylesheet" href="css/style.css">
  <style>
    .hidden {
      display: none;
    }
    .loading .display-loading {
      display: block;
    }
    .sampling .display-sampling {
      display: block;
    }
    .training .display-training {
      display: block;
    }
    .predicting .display-predicting {
      display: block;
    }
    #results {
      background-color: white;
      width: 20rem;
      min-height: 6rem;
      display: none;
      justify-content: center;
      align-items: center;
      font-weight: bold;
      font-size: 2rem;
    }
    .predicting #results {
      display: flex;
    }
  </style>
</head>
<body>
  <h1 class="h1">Neural Network Rock Paper Scissors</h1>
  <h2 id="state"></h2>
  <canvas id="canvas" class="fancy-shadow hidden display-sampling display-predicting display-training"></canvas>
  <div class="hidden display-sampling">
    <button id="rock">Rock</button>
    <button id="paper">Paper</button>
    <button id="scissors">Scissors</button>
  </div>
  <div class="hidden display-sampling">
    <button id="train">Train</button>
    <button id="load">Load the pretrained model</button>
  </div>
  <div class="hidden display-predicting">
    <button id="save">Save Model</button>
  </div>
  <div id="training" class="hidden display-training"></div>
  <div id="results" class="hidden display-predicting"></div>
  <script src="https://unpkg.com/ml5@1/dist/ml5.js"></script>
  <script>

    const $state = document.querySelector('#state');
    const $canvas = document.querySelector('#canvas');
    const $rock = document.querySelector('#rock');
    const $paper = document.querySelector('#paper');
    const $scissors = document.querySelector('#scissors');
    const $train = document.querySelector('#train');
    const $save = document.querySelector('#save');
    const $load = document.querySelector('#load');

    const $training = document.querySelector('#training');
    const $results = document.querySelector('#results');

    let video, ctx;
    let handPose;
    let classifier;
    let hands = [];
    let classificationResults = [];

    const STATE_LOADING = "loading";
    const STATE_SAMPLING = "sampling";
    const STATE_TRAINING = "training";
    const STATE_PREDICTING = "predicting";
    const ALL_STATES = [
      STATE_LOADING,
      STATE_SAMPLING,
      STATE_TRAINING,
      STATE_PREDICTING
    ];
    let state = STATE_LOADING;

    const setState = (value) => {
      console.log('setState', value);
      state = value;
      $state.textContent = state;
      document.documentElement.classList.remove(...ALL_STATES);
      document.documentElement.classList.add(state);
    };

    const preload = async () => {
      setState(STATE_LOADING);
      requestAnimationFrame(draw);
      console.log('preload');
      handPose = ml5.handPose();
      await handPose.ready;
      console.log('model ready');
      setup();
    }

    const setup = async () => {
      console.log('setup');
      ctx = $canvas.getContext('2d');
      // create a video stream - specify a fixed size
      const stream = await navigator.mediaDevices.getUserMedia({ video: {
        width: 640,
        height: 480
      } });
      video = document.createElement('video');
      video.srcObject = stream;
      video.play();
      // set canvas & video size
      $canvas.width = video.width = 640;
      $canvas.height = video.height = 480;
      // start detecting hands
      handPose.detectStart(video, (results) => {
        hands = results;
        if (state === STATE_PREDICTING) {
          if (hands.length === 0) {
            return;
          }
          // the keypoints should be one big array of numbers
          const keypoints = hands[0].keypoints.map(keypoint => [keypoint.x, keypoint.y]).flat();
          classifier.classify(keypoints, (results) => {
            classificationResults = results;
          });
        }
      });
      // For this example to work across all browsers
      // "webgl" or "cpu" needs to be set as the backend
      ml5.setBackend("webgl");
      // Set up the neural network
      let classifierOptions = {
        task: "classification",
        debug: true,
      };
      classifier = ml5.neuralNetwork(classifierOptions);
      const origin = new URL(window.location.href);
      const pretrainedModelURL = new URL("./models/rock-paper-scissors/model.json", origin);
      // add event listeners to buttons
      $rock.addEventListener('click', () => sample('rock'));
      $paper.addEventListener('click', () => sample('paper'));
      $scissors.addEventListener('click', () => sample('scissors'));
      $train.addEventListener('click', () => train());
      $save.addEventListener('click', () => classifier.save());
      $load.addEventListener('click', () => classifier.load(pretrainedModelURL.toString(), () => {
        console.log('model loaded');
        setState(STATE_PREDICTING);
      }));

      setState(STATE_SAMPLING);
    }

    const sample = (label) => {
      if (hands.length === 0) {
        return;
      }
      // the keypoints should be one big array of numbers
      const keypoints = hands[0].keypoints.map(keypoint => [keypoint.x, keypoint.y]).flat();
      classifier.addData(keypoints, [label]);
    };

    const train = () => {
      classifier.normalizeData();
      const options = {
        epochs: 50
      };
      classifier.train(options, whileTraining, finishedTraining);
      setState(STATE_TRAINING);
    };

    const whileTraining = (epoch, loss) => {
      $training.textContent = `Epoch: ${epoch}, Loss: ${loss.loss}`;
    };

    const finishedTraining = () => {
      console.log('finished training');
      setState(STATE_PREDICTING);
    };

    const draw = () => {
      if (state == STATE_LOADING) {
        drawLoading();
      } else if (state === STATE_SAMPLING) {
        drawSampling();
      } else if (state === STATE_TRAINING) {
        drawTraining();
      } else if (state === STATE_PREDICTING) {
        drawPredicting();
      }
      requestAnimationFrame(draw);
    }

    const drawLoading = () => {
    };

    const drawSampling = () => {
      drawVideoWithKeyPoints();
    };

    const drawTraining = () => {
      drawVideoWithKeyPoints();
    };

    const drawPredicting = () => {
      drawVideoWithKeyPoints();
      $results.textContent = classificationResults[0]?.label;
    };

    const drawVideoWithKeyPoints = () => {
      ctx.drawImage(video, 0, 0, $canvas.width, $canvas.height);
      ctx.fillStyle = 'red';
      hands.forEach(hand => {
        hand.keypoints.forEach(keypoint => {
          // no confidence score for handPose
          ctx.beginPath();
          ctx.arc(keypoint.x, keypoint.y, 10, 0, 2 * Math.PI);
          ctx.fill();
        });
      });
    };

    preload();
  </script>
</body>
</html>