Image Classifier - Hand Shadow Puppet AI Application using p5.js

Overview

This week, I focused on creating a simple Shadow Puppet AI application using p5.js and the Handpose model from ml5.js. The application captures live video footage of a person’s hand through a camera, detects the key points of the hand, and then draws a shadow-like representation of the hand on a canvas. The result is an interactive art piece where the user’s hand movements are mirrored on the screen as a shadow puppet.

Application Development

Hand Tracking with Handpose

The core functionality of the application is built using the Handpose model from ml5.js, which is designed for hand tracking. This model identifies key points on the hand, such as the main joints, and uses these points to map the hand’s skeletal structure.

1. Video Capture: The application begins by capturing live video footage from the user’s camera.
2. Hand Detection: The Handpose model detects the user’s hand in real-time, identifying key joint areas.
3. Drawing the Hand: The detected points are drawn onto a plain canvas as ellipses. Lines are then drawn between specific points to create the skeletal structure of the hand.
4. Shadow Effect: To enhance the visual representation, I modified the thickness of the lines and adjusted the alpha transparency value for the colours. This makes the drawn hand structure resemble a shadow much better.

Enhancements and Challenges

1. Mirroring the Hand: To improve interactivity, I flipped the video feed so that the drawn hand on the canvas would mirror the user’s real hand movements. This makes the application more intuitive and responsive.
2. Line Thickness and Transparency: I increased the line thickness to make the hand more visible on the screen and adjusted the transparency to enhance the shadow effect.
3. Avoiding Line Overlap: I attempted to modify the hand drawing so that the lines wouldn’t overlap, but encountered several errors. As a result, I reverted to the simpler structure.

Overall, this project was both interesting and challenging. It provided valuable insights into real-time hand tracking and interactive AI art applications. It would be fun to add this into a future project and maybe make the program recognise different animals that people sign with their hands.

Code

let model, video, keypoints, predictions = [];
let gfx;

function preload() {
  video = createCapture(VIDEO, () => {
    loadHandTrackingModel();
  });
  video.hide();
}

function setup() {
  const canvas = createCanvas(600, 400);
  canvas.parent('canvasContainer');
  gfx = createGraphics(600, 400);  // create a graphics buffer

}

async function loadHandTrackingModel() {
  // Load the MediaPipe handpose model.
  model = await handpose.load();
  select('#status').html('Hand Tracking Model Loaded')
  predictHand();
}

function draw() {
  background(260,250,200);
  
  //Mirror video
  translate(width, 0);
  scale(-1,1); //flip -1 scaling so horizontally
  if (predictions.length > 0) {
    drawKeypoints();
    drawSkeleton();
  }
}

async function predictHand() {
  // Pass in a video stream (or an image, canvas, or 3D tensor) to obtain a hand prediction from the MediaPipe graph.
  predictions = await model.estimateHands(video.elt);

  setTimeout(() => predictHand(), 100);
}

// A function to draw ellipses over the detected keypoints
function drawKeypoints() {
  let prediction = predictions[0];
  for (let j = 0; j < prediction.landmarks.length; j++) {
    let keypoint = prediction.landmarks[j];
    fill(32, 32, 32);
    strokeWeight(36);
    ellipse(keypoint[0], keypoint[1], 0, 0);
  }
}

// A function to draw the skeletons
function drawSkeleton() {
  let annotations = predictions[0].annotations;
  stroke(32, 32, 32, 180);
  strokeWeight(36);
  for (let j = 0; j < annotations.thumb.length - 1; j++) {
    line(annotations.thumb[j][0], annotations.thumb[j][1], annotations.thumb[j + 1][0], annotations.thumb[j + 1][1]);
  }
  for (let j = 0; j < annotations.indexFinger.length - 1; j++) {
    line(annotations.indexFinger[j][0], annotations.indexFinger[j][1], annotations.indexFinger[j + 1][0], annotations.indexFinger[j + 1][1]);
  }
  for (let j = 0; j < annotations.middleFinger.length - 1; j++) {
    line(annotations.middleFinger[j][0], annotations.middleFinger[j][1], annotations.middleFinger[j + 1][0], annotations.middleFinger[j + 1][1]);
  }
  for (let j = 0; j < annotations.ringFinger.length - 1; j++) {
    line(annotations.ringFinger[j][0], annotations.ringFinger[j][1], annotations.ringFinger[j + 1][0], annotations.ringFinger[j + 1][1]);
  }
  for (let j = 0; j < annotations.pinky.length - 1; j++) {
    line(annotations.pinky[j][0], annotations.pinky[j][1], annotations.pinky[j + 1][0], annotations.pinky[j + 1][1]);
  }

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.thumb[0][0], annotations.thumb[0][1]);
  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.indexFinger[0][0], annotations.indexFinger[0][1]);
  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.middleFinger[0][0], annotations.middleFinger[0][1]);
  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.ringFinger[0][0], annotations.ringFinger[0][1]);
  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.pinky[0][0], annotations.pinky[0][1]);
}

Reflection

This week’s project demonstrated the potential of combining hand tracking with creative coding. Although I faced challenges with overlapping lines and error handling, the final outcome successfully mirrored the user’s hand movements on the screen, creating a unique shadow puppet effect.

Video Explanation

Below is a short video where I walk through the code, demonstrate the shadow puppet application in action, and discuss the challenges encountered during development.