Category: 05-Body

chaine-Body

Idle:

Blinking:

Mouth open:

My first idea (scratched):

Idea that I pursued:

Starting off this project, I was debating between two ideas. One where I would play around with rotations, and the other where I would manipulate the eyes depending on the facial expression. (especially the mouth) I decided that I would like to express emotions creatively, so I decided to pursue that. At first, I wanted the face to express sadness by crying whenever the mouth opened and the eyebrows were furrowed, but I had some issues with creating the tears to be more fluid-like, so it kind of just looked like a scary person shooting out spherical beams. I got some new ideas with that mistake; and I thought it would be cool for the person to transform into an evil medusa with red pupils when you made a scary face. Also, I ended up removing the mouth entirely because the focus was in the eyes. Aesthetically speaking, I wish I played around with the eyes' outlines more because I think it's quite distracting especially since none of the other shapes have an outline. I also wish I could diversify the eyes' shapes to make it more evil-looking and scary. (possibly?) Functionally, I wish the eyes would have shot out beams rather than circles or better conveyed the user being frozen by her stone-turning eye beams. In terms of the relationship between the facial motions and the treatment of it, I would say that the motions came first and I only adapted to the possibilities of those motions.

//
// a template for receiving face tracking osc messages from
// Kyle McDonald's FaceOSC https://github.com/kylemcdonald/ofxFaceTracker
//
// 2012 Dan Wilcox danomatika.com
// for the IACD Spring 2012 class at the CMU School of Art
//
// adapted from from Greg Borenstein's 2011 example
// http://www.gregborenstein.com/
// https://gist.github.com/1603230
//
import oscP5.*;
OscP5 oscP5;
 
 
//I used Dan Shiffman's box2d adaptation: https://github.com/shiffman/Box2D-for-Processing
import shiffman.box2d.*;
import org.jbox2d.collision.shapes.*;
import org.jbox2d.common.*;
import org.jbox2d.dynamics.*;
import org.jbox2d.dynamics.joints.*;
 
Box2DProcessing box2d;
 
ArrayList<Boundary> boundaries;
ArrayList<ParticleSystem> systems;
ArrayList<Box> boxes;
 
// num faces found
int found;
 
// pose
float poseScale;
PVector posePosition = new PVector();
PVector poseOrientation = new PVector();
 
// gesture
float mouthHeight;
float mouthWidth;
float eyeLeft;
float eyeRight;
float eyebrowLeft;
float eyebrowRight;
float jaw;
float nostrils;
 
PImage outerEye;
 
float aVelocity = .05;
boolean mouseVelocity = false;
float angle = 0;
float amplitudeX = 200;
float amplitudeY = 200;
float theta = 0;
PVector location;
float centerX;
float centerY;
 
void setup() {
  size(640, 480);
  frameRate(30);
  smooth();
 
  box2d = new Box2DProcessing(this);
  box2d.createWorld();
 
  box2d.setGravity(0, -20);
 
  systems = new ArrayList<ParticleSystem>();
  boundaries = new ArrayList<Boundary>();
  boxes = new ArrayList<Box>();
 
  oscP5 = new OscP5(this, 8338);
  oscP5.plug(this, "found", "/found");
  oscP5.plug(this, "poseScale", "/pose/scale");
  oscP5.plug(this, "posePosition", "/pose/position");
  oscP5.plug(this, "poseOrientation", "/pose/orientation");
  oscP5.plug(this, "mouthWidthReceived", "/gesture/mouth/width");
  oscP5.plug(this, "mouthHeightReceived", "/gesture/mouth/height");
  oscP5.plug(this, "eyeLeftReceived", "/gesture/eye/left");
  oscP5.plug(this, "eyeRightReceived", "/gesture/eye/right");
  oscP5.plug(this, "eyebrowLeftReceived", "/gesture/eyebrow/left");
  oscP5.plug(this, "eyebrowRightReceived", "/gesture/eyebrow/right");
  oscP5.plug(this, "jawReceived", "/gesture/jaw");
  oscP5.plug(this, "nostrilsReceived", "/gesture/nostrils");
 
 
  outerEye = loadImage("circlebig.png");
 
  //boundaries.add(new Boundary(0,490,1280,10,0));
 
}
 
void draw() {  
  semiTransparent();
 
  box2d.step();
 
  for (ParticleSystem system: systems) {
    system.run();
 
    int n = (int) random(0, 2);
    system.addParticles(n);
  }
 
  for (Boundary wall: boundaries) {
    wall.display();
  }
 
  float varVelocity = calcVelocity(aVelocity);
  PVector angularVelocity = new PVector (angle, varVelocity);
  PVector amplitude = new PVector (amplitudeX, amplitudeY);
  PVector location = calculateCircle(angularVelocity, amplitude);
  //PVector centerCircle = calculateCenter(centerX, centerY);
 
  pushMatrix();
  if(found > 0) {
    drawOscillatingX(location);
  }
  popMatrix();
 
  for (Box b: boxes) {
    b.display();
  }
}
 
void semiTransparent() {
  rectMode(CORNER);
  noStroke();
  float backColor = map (mouthHeight, 1, 5, 255, 0);
  fill(backColor, backColor, backColor, 40);
  rect(0,0, width, height);
  stroke(0);
  noFill();
}
 
//basics of eye blink, iris movement from: https://raw.githubusercontent.com/jayjaycody/ComputationalCameras/master/wk3_Face/jai_face_keyPressComplexity/jai_face_keyPressComplexity.pde
 
float calcVelocity(float aVelocity) {
  float velocity = aVelocity;
  if (mouseVelocity == false) {
  }
  if (mouseVelocity == true) {
    velocity = map(mouseX, 0, width, -1, 1);
  }
  return velocity;
}
 
PVector calculateCircle (PVector angularVelocity, PVector amplitude) {
  float x = amplitude.x * cos (theta);
  float y = amplitude.y * sin (theta);
  location = new PVector (x, y);
  theta += angularVelocity.y;
  return location;
}
 
PVector calculateCenter (float centerX, float centerY) {
  PVector centerCircle = new PVector (centerX, centerY);
  return centerCircle;
}
 
void drawOscillatingX (PVector location) {
 
    float mouthScalar = map(mouthWidth, 10, 18, 0, 1.5); // make a scalar for location.x as a function of mouth
    location.mult(mouthScalar);
 
    float newPosX = map (posePosition.x, 0, 640, 0, width);
    float newPosY = map(posePosition.y, 0, 480, 0, height);  
    translate(width - newPosX, newPosY-100);
    scale(poseScale*.3);
    float irisColR = map (mouthHeight, 1, 5, 102, 204);
    float irisColG = map (mouthHeight, 1, 5, 204, 51);
    float irisColB = map (mouthHeight, 1, 5, 255, 0);
 
    float leftEyeMove = map(location.x, - amplitudeX, amplitudeX, -25, 33);
    pushMatrix();
    translate (leftEyeMove, 0);
    //Left iris
    fill(irisColR, irisColG, irisColB);
    noStroke();
 
    float eyeMult = map (mouthHeight, 1, 5, 1, 2);
 
    float irisSizeL = map (eyeLeft, 2, 3.5, 0, 50);
    ellipse(-100, 0, irisSizeL * eyeMult, irisSizeL * eyeMult);
 
    ////LeftPupil
    float eyeOutlineCol = map (mouthHeight, 1, 5, 0, 255);
 
    popMatrix();
 
    float rightEyeMove = map(location.x, - amplitudeX, amplitudeX, -33, 25);
    pushMatrix();   
    translate(rightEyeMove, 0);
    //right EYE
    //Right Iris
    fill(irisColR, irisColG, irisColB);
    noStroke();
 
    float irisSizeR = map (eyeRight, 2, 3.5, 0, 50);
    ellipse(100, 0, irisSizeR * eyeMult, irisSizeR * eyeMult);
 
    //Right Pupil
    stroke(eyeOutlineCol); 
    popMatrix();
    noFill();
 
 
    //get eye informatio and set scalar
    float blinkAmountRight = map (eyeRight, 2.5, 3.8, 0, 125);
    float blinkAmountLeft = map (eyeLeft, 2.5, 3.8, 0, 125);
 
 
    float eyeMultiplier = map (mouthHeight, 1, 5, 1, 3);
    // right eye size, blink and movement
    ellipse (100, 0, amplitudeX *.6, blinkAmountRight * eyeMultiplier); //scalar added to eyeHeight
    if (eyeRight < 2.7) {
      fill(255, 230, 204);
      ellipse (100, 0, amplitudeX *.6, blinkAmountRight*1.6 * (4 * eyeMultiplier/5)); //scalar added to eyeHeight
      noFill();
    }
 
    //left eye size, blink, and movement
    ellipse (-100, 0, amplitudeX *.6, blinkAmountLeft * eyeMultiplier); 
    if (eyeLeft < 2.7) {
      fill(255, 230, 204);
      ellipse (-100, 0, amplitudeX *.6, blinkAmountLeft*1.6 * (4 * eyeMultiplier/5)); //scalar added to eyeHeight
      noFill();
    }
 
    if (mouthHeight > 3.3) {
      //float mapScale = map (poseScale, 0, 4, 0, 1);
      pushMatrix();
      //translate(posePosition.x, posePosition.y);
      //scale(poseScale);
      Box p = new Box((width - posePosition.x - 100), (posePosition.y - 50));
      Box q = new Box((width - posePosition.x + 100), (posePosition.y - 50));
      boxes.add(p);
      boxes.add(q);
      popMatrix();
    }
 
}
 
public void found(int i) {
  println("found: " + i);
  found = i;
}
 
public void poseScale(float s) {
  println("scale: " + s);
  poseScale = s;
}
 
public void posePosition(float x, float y) {
  println("pose position\tX: " + x + " Y: " + y );
  posePosition.set(x, y, 0);
}
 
public void poseOrientation(float x, float y, float z) {
  println("pose orientation\tX: " + x + " Y: " + y + " Z: " + z);
  poseOrientation.set(x, y, z);
}
 
public void mouthWidthReceived(float w) {
  println("mouth Width: " + w);
  mouthWidth = w;
}
 
public void mouthHeightReceived(float h) {
  println("mouth height: " + h);
  mouthHeight = h;
}
 
public void eyeLeftReceived(float f) {
  println("eye left: " + f);
  eyeLeft = f;
}
 
public void eyeRightReceived(float f) {
  println("eye right: " + f);
  eyeRight = f;
}
 
public void eyebrowLeftReceived(float f) {
  println("eyebrow left: " + f);
  eyebrowLeft = f;
}
 
public void eyebrowRightReceived(float f) {
  println("eyebrow right: " + f);
  eyebrowRight = f;
}
 
public void jawReceived(float f) {
  println("jaw: " + f);
  jaw = f;
}
 
public void nostrilsReceived(float f) {
  println("nostrils: " + f);
  nostrils = f;
}
 
// all other OSC messages end up here
void oscEvent(OscMessage m) {
  if(m.isPlugged() == false) {
    println("UNPLUGGED: " + m);
  }
}

shuann-Body

Link to app: https://detailed-feet.glitch.me/

GIF:

Sketch:

My original idea was pretty different from what I eventually finished. Originally I wanted to create a drawing tool that one can draw with his/her pupils. The line would trace where the eye had been and the entire canvas would erase itself if the person blinks. However, I was having trouble to distinguish different state of the eyes with the face tracking templet code and decided to go to a different direction.

Then the new idea I had was to use the cereal head movement to control a basic character. I first thought of the more conventional multiple level of floors that a character could be placed into by change the vertical position of the head. However I realized that did not make much sense since I want to use head turn to trigger "jump". The left&right motion does not naturally map to the up&down motion of the jump. Therefore I redesigned the floor to be vertical and have the character run up the screen to avoid obstacles. That been said, this not a game yet as it does not have any level design, no goals to achieve, etc. As it for now, it's more like an exploration on the kind of controllers that can be implemented. If there is more time, I would like to turn this into a game that two people can play against each other at the same time.

Still Image:

 Video:

var local;
var obs = [];
var dir = [1, -1];
var player1 = null;
var stuck = false;
var score1 = 0;
var vel = 8;
 
var from;
var to;
var bgcX = 0;
 
function setup() {
 
  createCanvas(640, 480);
  background(0);
 
  //this is where posenet initlizes and webcam stuff happens
  PoseZero.init();
 
  //this is you, check out agent.js for adding properties and functions
  local = new Agent();
 
  player1 = new player(width/2,height*2/3,random(dir));
  obs[0] = new obstacle(width/2, height/2, random(15,21), random(dir));
  obs[1] = new obstacle(width/2, height/3, random(15,21), random(dir));
 
}
 
function draw() {
  background(255);
 
	local.update(PoseZero.get());// update your skeleton
 
  push();
  if (local.data.pose != null){ 
    push();
    if (player1.dir === 1){
      bgcX = width/2;
      from = color(255,100,147);
      to = color(255,182,193);
    }
    if (player1.dir === -1){
      bgcX = 0;
      from = color(255,182,193);
      to = color(255,100,147);
    }
    noStroke();
    fill(150);
    var widthRect = width/40;
    noStroke();
    for (var i=0;i&lt;20;i++){
      inter = lerpColor(from,to,(i+1)*(1/20));
      fill(inter);
      rect(bgcX + i*widthRect,0,widthRect,height);
    }
    pop();
 
    PoseZero.draw_pose(local.data.pose,{color:local.data.color})
 
    if (player1.dir === 1){
      text("shake your head RIGHT" , width/2 + 170, height - 20);
    }
    if (player1.dir === -1){
      text("shake your head LEFT ", 20, height - 20);
    }
 
    push();
    drawPlayground();
 
    palceObs();
    boundaryCheck();
    speedCheck();
 
    if (player1.stuck() === false){
      for (var elem = 0; elem &lt; obs.length; elem ++){
        obs[elem].draw();
        obs[elem].move();
      }
    } else {
      for (var elem = 0; elem &lt; obs.length; elem ++){ obs[elem].draw(); } } player1.draw(); player1.checkDir(); player1.headTurn(); pop(); } pop(); } function palceObs(){ if (frameCount % 15 == 0){ if (obs[obs.length - 1].y != -20){ obs.push(new obstacle(width/2, -20, random(15,21), random(dir))); } } } function boundaryCheck(){ if (obs[0].y &gt;= height){
    obs.shift();
  }  
}
 
function speedCheck(){
  var closeness = PoseZero.estimate_scale(local.data.pose);
 
  if (closeness &lt;= 70 &amp;&amp; closeness &gt; 40) {
    vel = 8;
  }
 
  if (closeness &lt;= 100 &amp;&amp; closeness &gt; 70) {
    vel = 11;
  }
 
  if (closeness &lt;= 120 &amp;&amp; closeness &gt; 100) {
    vel = 14;
  }
}
 
function drawPlayground(){
  var road = width/2;
  strokeWeight(4);
  line(road, 0, road, height);
 
}
 
function obstacle(x,y,s,dir){
  this.x = x;
  this.y = y;
  this.dir = dir;
  this.s = s;
  this.r = int(random(240,255));
	this.g = int(random(30,200));
	this.b = int(random(50,69));
 
  this.draw = function(){
    push();
    stroke(0);
		strokeWeight(4);
		fill(this.r,this.g,this.b);
    if (this.dir == 1){
      rect(this.x, this.y, this.s * 4, 20, 30);
      for (var a=0; a&lt;4; a++){
        push();
        stroke(250);
			  line(this.x + 6 + 15*a, this.y + 3, this.x + 15*a + 15, this.y + 17);
        pop();
		  }
    } else {
      rect(this.x - this.s * 4, this.y, this.s * 4, 20, 30);
      for (var b=0; b&lt;4; b++){
        push();
        stroke(250);
			  line(this.x + 6 + 15*b - this.s * 4, this.y + 3, this.x + 15*b + 15 - this.s * 4, this.y + 17);
        pop();
		  }
    }
    pop();
  }
 
  this.move = function(){
    this.y += vel;
  } 
}
 
function player(x,y,dir){
  this.x = x;
  this.y = y;
  this.dir = dir;
  this.h = 50;
  this.w = 30;
  this.leg = 15;
  this.vel = 10;
  var triX;
  var jumpR = false;
  var toppedR = false;
  var jumpL = false;
  var toppedL = false;
 
  this.draw = function(){
    // console.log(this.dir);
    if (this.dir == -1) {
      triX = this.x - this.leg;
    } else {
      triX = this.x + this.leg;
    }
    push();
    strokeJoin(ROUND);
    strokeWeight(4);
    stroke(255);
    fill(255, 209, 124);
    line(triX + this.h*3/4 * this.dir, this.y + this.w/2, this.x + this.dir * 5, this.y + this.w/2);
    triangle(triX, this.y, triX + this.h * this.dir, this.y + this.w/2, triX, this.y + this.w);
    ellipse(triX + this.h*3/4 * this.dir, this.y + this.w/2, this.leg*2 , this.leg*2);
    stroke(0);
    line(triX + (this.h*7/8 - 4) * this.dir, this.y + this.w/2 - 6, triX + this.h*6/8 * this.dir, this.y + this.w/2 - 6);
    line(triX + (this.h*7/8 - 4) * this.dir, this.y + this.w/2 + 6, triX + this.h*6/8 * this.dir, this.y + this.w/2 + 6);
    pop();
  }
 
  this.checkDir = function(){
    var playerPos = PoseZero.nosePosX(local.data.pose);
 
    if (playerPos &lt; width/2) {
      this.dir = -1;
    } else {
      this.dir = 1;
    }
  }
 
  this.headTurn = function(){
    //check for jumps
    if (PoseZero.rightTurn(local.data.pose) &amp;&amp; this.dir == 1){
      jumpR = true;
    }
    if (PoseZero.leftTurn(local.data.pose) &amp;&amp; this.dir == -1){
      jumpL = true;
    }
 
    if (jumpR === true){
      this.jumpRight();
    } 
    if (jumpL === true){
      this.jumpLeft();
    }
  }
 
  //Right jump animation
  this.jumpRight = function(){
    if (toppedR === false &amp;&amp; this.x &lt; width/2 + 100){ this.x += this.vel; if (this.x &gt;= width/2 + 100) {
        toppedR = true;
      }
    } 
    if (toppedR === true &amp;&amp; this.x != width/2){
      this.x -= this.vel;
      if (this.x &lt;= width/2) { this.x = width/2; jumpR = false; toppedR = false; } } } //Left jump animation this.jumpLeft = function(){ if (toppedL === false &amp;&amp; this.x &gt; width/2 - 100){
      this.x -= this.vel;
      if (this.x &lt;= width/2 - 100) { toppedL = true; } } if (toppedL === true &amp;&amp; this.x != width/2){ this.x += this.vel; if (this.x &gt;= width/2) {
        this.x = width/2;
        jumpL = false;
        toppedL = false;
      }
    }
  }
 
  this.stuck = function(){
  //check for obstacles
    for (var elem = 0; elem &lt; obs.length; elem ++){
      var obsTop = obs[elem].y;
      var obsBottom = obs[elem].y + 20;
      var obsOuter = obs[elem].x + obs[elem].s * obs[elem].dir * 4;
      var obsInner = obs[elem].x;
      if (obs[elem].dir == this.dir &amp;&amp; obsTop &lt; this.y &amp;&amp; obsBottom &gt;= this.y){
        if (this.dir === 1 &amp;&amp; this.x &lt;= obsOuter){ return true; } if (this.dir === -1 &amp;&amp; this.x &gt;= obsOuter){
          return true;
        }
      }
    }
    return false;
  }
}

 

nixel-Body

Video:

Gif:

Description:

This project came naturally to me since I'm interested in dance and videography. From the beginning, I was curious about how I could convey movement, choreography, and group dynamics with the motion capture resources provided.  I wanted to study how choreography that relied on more than one dancer created compositions with bodies and movements on screen.

Process:

I went through several iterations of how I could convey what I wanted to, so after figuring out how Posenet worked, I ended up making a lot of visual representations of bodies in movement using some of my favorite dance videos. A lot of these I trashed because they looked too clunky and didn't feel right. It felt eh getting rid of several hundred lines of usable material, but in the end, I decided that visual simplicity and readability was more important than the technical work behind it.

I was not aiming for accuracy with this project, and I was okay with an end result that was not so minutely tied to the body in particular since I wanted to visualize several bodies in relation to each other. I wanted some sort of big picture, wholistic experience rather than something dependent on accuracy. Early on, I found that Posenet would never be able to capture the bodies as well as I wanted to, so I decided to use that messiness as an advantage. I kind of like how it sometimes mistakes body parts for each other, or sees a phantom body part in the wall somewhere.

The first, second, and fourth video in the compilation are all choreographies/videographies that I love a lot. The third video is one that I filmed for KPDC, a dance club on campus. I was interested to see how a video that I produced compared to those that I admired.

Still Images:

Code:

// Copyright (c) 2018 ml5
// Copyright (c) 2018 ml5
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
 
/* ===
ml5 Example
PoseNet example using p5.js
=== */
 
// PoseNet with a pre-recorded video, modified from:
// https://github.com/ml5js/ml5-examples/blob/master/p5js/PoseNet/sketch.js
 
let poseNet;
let poses = [];
let poseArray = [];
let noseArray = [];
let leftWristArray = [];
let rightWristArray = [];
let rightAnkleArray = [];
 
let video;
var videoIsPlaying;
 
function setup() {
  frameRate(10);
  videoIsPlaying = false;
  createCanvas(833, 480);
  video = createVideo('yabbay.mp4', vidLoad);
  video.size(width, height);
  poseNet = ml5.poseNet(video, modelReady);
  poseNet.on('pose', function(results) {
    poses = results;
    if(poses.length &gt; 0){
      poseArray.push(poses);
    }
  });
 
  video.hide();
}
 
function modelReady() {
    console.log('Model Loaded');
}
 
function mousePressed(){
  vidLoad();
}
 
function draw() {
  image(video, 0, 0, width, height);
  //background(0);
  drawKeypoints();
}
 
function drawKeypoints()  {
//   for (let i = 0; i &lt; poses.length; i++) {
//
//     let pose = poses[i].pose;
//
//     for (let j = 2; j &lt; pose.keypoints.length; j=j+10) {
//       for (let k = 3; k &lt; pose.keypoints.length; k=k+10){
//         for (let l = 4; l &lt; pose.keypoints.length; l=l+10){
//           for (let m = 5; m &lt; pose.keypoints.length; m=m+10){ // let keypoint = pose.keypoints[j]; // let keypoint2 = pose.keypoints[k]; // let keypoint3 = pose.keypoints[l]; // let keypoint4 = pose.keypoints[m]; // // if (keypoint.score &gt; 0.2) {
//         strokeWeight(1);
//         stroke(255,0,0);
//
//         beginShape();
//         curveVertex(keypoint.position.x, keypoint.position.y);
//         curveVertex(keypoint2.position.x, keypoint2.position.y);
//         curveVertex(keypoint3.position.x, keypoint3.position.y);
//         curveVertex(keypoint4.position.x, keypoint4.position.y);
//         endShape();
//       }
//     }
//   }
//   }
// }
// }
 
  for (var i = 0; i &lt; poses.length; i++){
    let pose = poses[i].pose;
    for (var j = 0; j &lt; pose.keypoints.length; j++){ noFill(); strokeWeight(1); makeVectors("nose", pose, j, noseArray); makeVectors("leftWrist", pose, j, leftWristArray); makeVectors("rightAnkle", pose, j, rightAnkleArray); stroke(255); makeConnections(leftWristArray); makeConnections(noseArray); makeConnections(rightAnkleArray); } } } function makeVectors(part, pose, j, partArray){ if(pose.keypoints[j].part == part){ stroke(0, 0, 255); ellipse(pose.keypoints[j].position.x,pose.keypoints[j].position.y, 10); partArray.push(createVector(pose.keypoints[j].position.x,pose.keypoints[j].position.y)); if (partArray.length &gt; 4){
      partArray.splice(0,1);
    }
  }
}
 
function makeConnections(partArray){
  beginShape();
  for (let k = 0; k &lt; partArray.length; k++){
    curveVertex(partArray[k].x, partArray[k].y);
  }
  endShape();
}
 
function vidLoad() {
  video.stop();
  video.loop();
  videoIsPlaying = true;
}
 
function keyPressed(){
  if (videoIsPlaying) {
    video.pause();
    videoIsPlaying = false;
  } else {
    video.loop();
    videoIsPlaying = true;
  }
}

lass-Body

Each of the figures is created randomly using the names of people from this class:

You can play with the random generation tool here:

For this project, I wanted to randomly create humanoid figures that look like they are made of garbage. I am pretty satisfied with the final product, and I think a lot of the success is thanks to these nice color palettes. I also got all of my bvh files from the CMU Graphics Lab. Finally, I used seedrandom so that each random figure is dependent on its name. I was very inspired by Generative Machines by Michael Chang, and other generative artworks shown in class.

Here are my sketches:

 

    //starter code for loading BVH from https://github.com/mrdoob/three.js/blob/master/examples/webgl_loader_bvh.html
    var clock = new THREE.Clock();
    var camera, controls, scene, renderer;
    var mixer, skeletonHelper, boneContainer;
    var head, lhand, rhand, torso, rfoot, lfoot; 
    var miscParts = []; 
    var limbs = [
    new THREE.BoxGeometry( 10, 10, 10 ), 
    new THREE.SphereGeometry( 10, 20, 15 ), 
    new THREE.ConeGeometry( 10, 10, 30 ), 
    new THREE.CylinderGeometry( 10, 10, 10, 5 ), 
    new THREE.TorusGeometry( 7, 3, 10, 30 ), 
    new THREE.TorusKnotGeometry( 7, 3, 10, 30 ), 
    new THREE.DodecahedronGeometry(7)
]
var bodies = [
    new THREE.BoxGeometry( 30, 60, 30 ), 
    new THREE.SphereGeometry( 30, 20, 15 ), 
    new THREE.ConeGeometry( 30, 60, 30 ), 
    new THREE.CylinderGeometry( 20, 30, 50, 5 ), 
    new THREE.TorusGeometry( 20, 10, 10, 30 ), 
    new THREE.TorusKnotGeometry( 20, 10, 10, 30 ), 
    new THREE.DodecahedronGeometry(20)
]
var colorArray = [new THREE.Color(0xffaaff), new THREE.Color(0xffaaff),  new THREE.Color(0xffaaff),  new THREE.Color(0xffaaff),  new THREE.Color(0xffaaff)]; 
var uniforms = {u_resolution: {type: "v2", value: new THREE.Vector2()}, 
                u_colors: {type: "v3v", value: colorArray}};
 
var materials = [
    new THREE.ShaderMaterial( {  
        uniforms: uniforms,
        vertexShader: document.getElementById("vertex").textContent,
        fragmentShader: document.getElementById("stripeFragment").textContent,
    }), 
    new THREE.ShaderMaterial( {  
        uniforms: uniforms,
        vertexShader: document.getElementById("vertex").textContent,
        fragmentShader: document.getElementById("gradientFragment").textContent,
    }), 
    new THREE.ShaderMaterial( {  
        uniforms: uniforms,
        vertexShader: document.getElementById("vertex").textContent,
        fragmentShader: document.getElementById("plainFragment").textContent
    }), 
    new THREE.ShaderMaterial( {  
        uniforms: uniforms,
        vertexShader: document.getElementById("vertex").textContent,
        fragmentShader: document.getElementById("plainFragment").textContent,
        wireframe: true 
    })
    ]
 
    init();
    animate();
 
    var bvhs = ["02_04", "02_05", "02_06", "02_07", "02_08", "02_09", "02_10", "pirouette"]
 
    uniforms.u_resolution.value.x = renderer.domElement.width;
    uniforms.u_resolution.value.y = renderer.domElement.height;
 
    var loader = new THREE.BVHLoader();
    loader.load( "models/" + random(bvhs) + ".bvh", createSkeleton);
 
    function createSkeleton(result){
        skeletonHelper = new THREE.SkeletonHelper( result.skeleton.bones[ 0 ] );
        skeletonHelper.skeleton = result.skeleton; // allow animation mixer to bind to SkeletonHelper directly
        boneContainer = new THREE.Group();
 
        boneContainer.add( result.skeleton.bones[ 0 ] );
        var geometry = new THREE.BoxGeometry( 10, 10, 10 );
        head = new THREE.Mesh( random(limbs), materials[0] );
        lhand = new THREE.Mesh( random(limbs), materials[0] );
        rhand = new THREE.Mesh( random(limbs), materials[0] );
        lfoot = new THREE.Mesh( random(limbs), materials[0] );
        rfoot = new THREE.Mesh( random(limbs), materials[0] );
        torso = new THREE.Mesh( random(bodies), materials[0] );
        // torso.scale.set(Math.random() * 1.5, Math.random() * 1.5, Math.random() * 1.5);
 
        skeletonHelper.skeleton.bones[4].add(head); 
        skeletonHelper.skeleton.bones[12].add(rhand); 
        skeletonHelper.skeleton.bones[31].add(lhand); 
        skeletonHelper.skeleton.bones[50].add(rfoot); 
        skeletonHelper.skeleton.bones[55].add(lfoot); 
        skeletonHelper.skeleton.bones[1].add(torso); 
        for(var i=9; i<14; i++){
            var part = new THREE.Mesh(  new THREE.BoxGeometry( Math.random() * 10, Math.random() * 5, Math.random() * 5 ), materials[0] ); 
            miscParts.push(part); 
            skeletonHelper.skeleton.bones[i].add(part);
        }
        for(var i=28; i<31; i++) {
            var part = new THREE.Mesh(  new THREE.BoxGeometry( Math.random() * 10, Math.random() * 5, Math.random() * 5 ), materials[0] ); 
            miscParts.push(part); 
            skeletonHelper.skeleton.bones[i].add(part);
        }        
        for(var i=47; i<56; i++) {
            var part = new THREE.Mesh(  new THREE.BoxGeometry( Math.random() * 10, Math.random() * 5, Math.random() * 5 ), materials[0] ); 
            miscParts.push(part); 
            skeletonHelper.skeleton.bones[i].add(part);
        }
        scene.add( skeletonHelper );
        scene.add( boneContainer );
        skeletonHelper.material = new THREE.MeshBasicMaterial({
            color:"white", 
            transparent:"true", 
            opacity:"0.0"}); 
        mixer = new THREE.AnimationMixer( skeletonHelper );
        mixer.clipAction( result.clip ).setEffectiveWeight( 1.0 ).play();
        changeName(); 
    }
    function init() {
        camera = new THREE.PerspectiveCamera( 60, window.innerWidth / window.innerHeight, .1, 1000 );
        camera.position.set( 0, 200, 400 );
        scene = new THREE.Scene();
        scene.add( new THREE.GridHelper( 400, 10 ) );
        scene.background = new THREE.Color(0xdddddd); 
        renderer = new THREE.WebGLRenderer( { antialias: true } );
        renderer.setPixelRatio( window.devicePixelRatio );
        renderer.setSize( window.innerWidth, window.innerHeight );
        document.body.appendChild( renderer.domElement );
        controls = new THREE.OrbitControls( camera, renderer.domElement );
        controls.minDistance = 300;
        controls.maxDistance = 700;
        window.addEventListener( 'resize', onWindowResize, false );
    }
    function onWindowResize() {
        camera.aspect = window.innerWidth / window.innerHeight;
        camera.updateProjectionMatrix();
        renderer.setSize( window.innerWidth, window.innerHeight );
    }
    function animate() {
        requestAnimationFrame( animate );
        var delta = clock.getDelta();
        if ( mixer ) mixer.update( delta );
        renderer.render( scene, camera );
    }
    function changeBody(seed) {
        console.log(seed); 
        Math.seedrandom(seed); 
 
        color = random(colors); 
        for(var i = 0; i < 5; i++){
            colorArray[i] = new THREE.Color(color[i]); 
        }
        for(var i = 0; i < materials.length; i++){
            materials[i].clone(); //idk why i do this but its the only way to make randomness match with class.html
        }
        lhand.geometry = random(limbs); 
        lhand.material = random(materials); 
        rhand.geometry = random(limbs); 
        rhand.material = random(materials); 
        torso.geometry = random(bodies); 
        torso.material = random(materials); 
        lfoot.geometry = random(limbs); 
        lfoot.material = random(materials); 
        rfoot.geometry = random(limbs); 
        rfoot.material = random(materials);
        head.geometry = random(limbs); 
        head.material = random(materials); 
        head.scale.set(2, 2,2); 
        for(var i=0; i<miscParts.length; i++){
            miscParts[i].geometry =  new THREE.BoxGeometry( Math.random() * 15, Math.random() * 15, Math.random() * 10 );
            miscParts[i].material = random(materials); 
        }
    }    function changeName() {
        changeBody(document.getElementById("nameInput").value); 
    }
    function random(arr) {
        return arr[Math.floor(Math.random() * arr.length)];
    }
    function changeBvh(){
        scene.remove(skeletonHelper); 
        scene.remove(boneContainer); 
        loader.load( "models/"+ random(bvhs) +".bvh", createSkeleton);
    }

ocannoli-Body

Star Collector

Move your face to direct the telescope (the blue circle) to the special yellow stars. Once a star is in your sights, open your mouth to collect it and get a point. But be careful not to raise your eyebrows too high or the aliens will attack and take your points.

Going into this project, brainstorming for ideas was difficult for me. I had a lot of shallow, surface level ideas but no a lot of concepts I thought were good enough. A lot of my process started conceptually with an idea of the project or the narrative rather than understanding the full relationship with the body; therefore, I never fully landed on something I thought was interesting or in depth enough. For the star collector idea I liked the idea of the the ability for the body or even simply the face to control space or the cosmos. I wanted to add more features and a different style which would limit the players view to just that circle and have the board be much bigger. I also wanted players to find patterns, or maybe constellations rather than other "stars".  I used face OSC in processing which was both an intentional move but also somewhat guided the relationship between the body and the interface. Overall, I enjoyed the experience of working with motion capture for the first time and if I were to redo this project I'd like to come up with something more complex either conceptually or in execution.

 //
// a template for receiving face tracking osc messages from
// Kyle McDonald's FaceOSC https://github.com/kylemcdonald/ofxFaceTracker
//
// 2012 Dan Wilcox danomatika.com
// for the IACD Spring 2012 class at the CMU School of Art
//
// adapted from from Greg Borenstein's 2011 example
// http://www.gregborenstein.com/
// https://gist.github.com/1603230
//
import oscP5.*;
OscP5 oscP5;
 
// num faces found
int found;
 
// pose
float poseScale;
PVector posePosition = new PVector();
PVector poseOrientation = new PVector();
 
// gesture
float mouthHeight;
float mouthWidth;
float eyeLeft;
float eyeRight;
float eyebrowLeft;
float eyebrowRight;
float jaw;
float nostrils;
float circleX;
float circleY; 
PImage starSky;
PImage aliens; 
int score; 
FloatList specialStars;
PFont fjFont;
int numOfStars; 
boolean doneOnce;
 
 
 
void setup() {
  size(800, 800);
  frameRate(30);
 
  oscP5 = new OscP5(this, 8338);
  oscP5.plug(this, "found", "/found");
  oscP5.plug(this, "poseScale", "/pose/scale");
  oscP5.plug(this, "posePosition", "/pose/position");
  oscP5.plug(this, "poseOrientation", "/pose/orientation");
  oscP5.plug(this, "mouthWidthReceived", "/gesture/mouth/width");
  oscP5.plug(this, "mouthHeightReceived", "/gesture/mouth/height");
  oscP5.plug(this, "eyeLeftReceived", "/gesture/eye/left");
  oscP5.plug(this, "eyeRightReceived", "/gesture/eye/right");
  oscP5.plug(this, "eyebrowLeftReceived", "/gesture/eyebrow/left");
  oscP5.plug(this, "eyebrowRightReceived", "/gesture/eyebrow/right");
  oscP5.plug(this, "jawReceived", "/gesture/jaw");
  oscP5.plug(this, "nostrilsReceived", "/gesture/nostrils");
  starSky = loadImage("stars.png");
  aliens= loadImage("ALIENS.png");
  score=0; 
  specialStars = new FloatList();
  numOfStars=7;
  for (int i=0; i&lt;numOfStars;i++){ createSpecialStar(); } fjFont = createFont("FjallaOne-Regular.ttf",32); doneOnce=false; } public void createSpecialStar(){ float x= random(0,width); float y=random(85,height-180); specialStars.append(x); specialStars.append(y); } public void newSpecialStars(){ for (int i=specialStars.size()-1;i&gt;-1; i--){
     specialStars.remove(i);
   }
   for (int i=0; i&lt;numOfStars; i++){
     createSpecialStar(); 
   }
 }
 
void draw() {  
  image(starSky,0,0);
  int seconds = second();
  //refreshes special stars every 5 seconds
  if ((seconds-1)%5==0 &amp;&amp; doneOnce==true){
    doneOnce=false; 
  }
  if (seconds%5==0 &amp;&amp;doneOnce==false){
    newSpecialStars(); 
    doneOnce=true; 
 
  }
  //tint(255,255,255); 
  float radius=40; 
  textFont(fjFont);
  textSize(50); 
  text("Score: "+score,10,height-10); 
  fill(255,255,102);
  noStroke(); 
  //makes special stars
  for (int i=0; i&lt;specialStars.size(); i+=2){
    float starX=specialStars.get(i);
    float starY=specialStars.get(i+1);
    ellipse(starX, starY, 10,10);   
  }
 
  //Makes telescope Circle
  stroke(0);
  noFill(); 
  stroke(204,229,255);
  strokeWeight(4);
  ellipse(circleX,circleY,radius*2,radius*2); 
 
  for (int i=0; i&lt;specialStars.size(); i+=2){
    float starX=specialStars.get(i);
    float starY=specialStars.get(i+1);
    //makes stars then checks valid to eat star
    if (starX&lt;circleX+(radius/2) &amp;&amp; starX&gt;circleX-(radius/2) &amp;&amp; starY&lt;circleY+(radius/2) &amp;&amp; starY&gt;circleY-(radius/2)){ 
        if(jaw&gt;23.5){
          specialStars.remove(i+1);
          specialStars.remove(i);
          score+=1; 
          createSpecialStar(); 
        }
 
    }
  }
 
  if (eyebrowLeft&gt;9.2){
    image(aliens,width/6,height/4,592,326.5);
    if (score&lt;=5){ score=0; } else if (score&gt;5){
      score=score-5; 
    }
  }
 
}
 
 
 
// OSC CALLBACK FUNCTIONS
 
public void found(int i) {
  println("found: " + i);
  found = i;
}
 
public void poseScale(float s) {
  println("scale: " + s);
  poseScale = s;
}
 
public void posePosition(float x, float y) {
  println("pose position\tX: " + x + " Y: " + y );
  posePosition.set(x, y, 0);
  circleX=x+(posePosition.x-150);
  circleY=y+(posePosition.y-150);
 
 
}
 
public void poseOrientation(float x, float y, float z) {
  println("pose orientation\tX: " + x + " Y: " + y + " Z: " + z);
  poseOrientation.set(x, y, z);
}
 
public void mouthWidthReceived(float w) {
  println("mouth Width: " + w);
  mouthWidth = w;
}
 
public void mouthHeightReceived(float h) {
  println("mouth height: " + h);
  mouthHeight = h;
}
 
public void eyeLeftReceived(float f) {
  println("eye left: " + f);
  eyeLeft = f;
}
 
public void eyeRightReceived(float f) {
  println("eye right: " + f);
  eyeRight = f;
}
 
public void eyebrowLeftReceived(float f) {
  println("eyebrow left: " + f);
  eyebrowLeft = f;
}
 
public void eyebrowRightReceived(float f) {
  println("eyebrow right: " + f);
  eyebrowRight = f;
}
 
public void jawReceived(float f) {
  println("jaw: " + f);
  jaw = f;
}
 
public void nostrilsReceived(float f) {
  println("nostrils: " + f);
  nostrils = f;
}
 
// all other OSC messages end up here
void oscEvent(OscMessage m) {
  if(m.isPlugged() == false) {
    println("UNPLUGGED: " + m);
  }
}

 

airsun-body

Anxiety is the inspiration for the project. More specifically, reflecting on me often get nervous when people interacting with me. When people that I am not familiar with talk with me, sometimes, I just blushed and could not control it. Moreover, when talking to other introverts, I found we share a similar experience. Therefore, I want to portray this nervous tension going on between individuals, and exaggerate it. And this is when Ovia was born. And to help my audience understand more about her, here is a paragraph of the introduction: "Ovia is quiet and simple being. She is very shy and nervous when interacting with people. When you try to interact with her, she will always be shaking, blushing, and getting smaller. When you push her limits, she will hide all her features and become a ball."

// Clair Sijing Sun
// Copyright (c) 2018 ml5
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
 
 
let video;
let poseNet;
let poses = [];
var r=20;
var posy;
var posx;
var timer = 0;
var dim;
// var colorR = 252;
// var colorG = 238;
// var colorB = 213;
var colorR = 250;
var colorG = 210;
var moving = 1;
var hairp1x = 301;
var hairp1y = 136-40;
var hairp2x = 297;
var hairp2y = 129-40;
var hairp3x = 303;
var hairp3y = 118-40;
var hairp4x = 299;
var hairp4y = 115-40;
var right = 1;
var left  = 1;
var radius;
var blushl = 59;
var frames = [];
 
 
function setup() {
  createCanvas(640, 480);
  ellipseMode(RADIUS);
  dim = width/2;
  posx = width/2;
  radius = width/6;
  //colorMode(HSB, 360, 100, 100);
  posy = height/2-40;
  video = createCapture(VIDEO);
  video.size(width, height);
 
  // Create a new poseNet method with a single detection
  poseNet = ml5.poseNet(video, modelReady);
  // This sets up an event that fills the global variable "poses"
  // with an array every time new poses are detected
  poseNet.on('pose', function(results) {
    poses = results;
  });
  // Hide the video element, and just show the canvas
  video.hide();
}
 
function preload(){
  frames.push(loadImage("https://i.imgur.com/qVyg9s1.png"));
}
 
function modelReady() {
  select('#status').html('!!!');
}
 
function draw() {
  background(255);
 
  timer +=1;
 
  //background face
  if (poses.length &gt; 0){
    posx = random(width/2-moving,width/2+moving);
    moving = randomGaussian (-poses.length,poses.length);
 
  }
  if (poses.length &lt; 5){
    drawGradient(posx, posy);
  }
 
  //the eyewhite
  push();
  fill(252,238, 213);
  if (poses.length &lt; 5){ ellipse(posx,posy,20,20); } fill(0); drawKeypoints(); stroke(4); pop(); //blushing push(); noStroke(); if (poses.length == 1){ if (blushl &gt; 55.6){
     blushl -= 0.2;
    } if (blushl &lt; 55.6){ blushl += 0.2; } }if (poses.length == 2){ if (blushl &gt; 50){
     blushl -= 0.2;
    } if (blushl &lt; 50){ blushl += 0.2; } }if (poses.length == 3){ if (blushl &gt; 38){
     blushl -= 0.2;
    }if (blushl &lt; 38){ blushl += 0.2; } }if (poses.length == 4){ if (blushl &gt; 16){
     blushl -= 0.2;
    }if (blushl &lt; 16){
     blushl += 0.2;
    }
  }
 
  fill(255,126,0);
  if (poses.length &lt; 5){ ellipse(posx+blushl, 240, 10, 6); ellipse(posx-blushl, 240, 10, 6); } pop(); //hair drawHair(); image(frames[0], width/2-190, height/2+50, 400, 200); } function drawHair() { stroke(1); noFill(); beginShape(); if (timer%10 == 0){ right *= -1; } if (timer%15 == 0){ left *= -1; } hairp1x = posx - 19; hairp2x +=right; hairp3x -=left*0.5; hairp4x +=right; if (poses.length == 0){ if (hairp1y &gt; 95){
      hairp1y = hairp1y - 0.2;
      hairp2y = hairp2y - 0.2;
      hairp3y = hairp3y - 0.2;
      hairp4y = hairp4y - 0.2;
    }
  }
  if (poses.length == 1){
    if (hairp1y &lt; 101){ hairp1y = hairp1y + 0.2; hairp2y = hairp2y + 0.2; hairp3y = hairp3y + 0.2; hairp4y = hairp4y + 0.2; }if (hairp1y &gt; 101){
      hairp1y = hairp1y - 0.2;
      hairp2y = hairp2y - 0.2;
      hairp3y = hairp3y - 0.2;
      hairp4y = hairp4y - 0.2;
    }
  }if (poses.length == 2){
    if (hairp1y &lt; 111){ hairp1y = hairp1y + 0.2; hairp2y = hairp2y + 0.2; hairp3y = hairp3y + 0.2; hairp4y = hairp4y + 0.2; }if (hairp1y &gt; 111){
      hairp1y = hairp1y - 0.2;
      hairp2y = hairp2y - 0.2;
      hairp3y = hairp3y - 0.2;
      hairp4y = hairp4y - 0.2;
    }
  }if (poses.length == 3){
    if (hairp1y &lt; 131){ hairp1y = hairp1y + 0.2; hairp2y = hairp2y + 0.2; hairp3y = hairp3y + 0.2; hairp4y = hairp4y + 0.2; }if (hairp1y &gt; 131){
      hairp1y = hairp1y - 0.2;
      hairp2y = hairp2y - 0.2;
      hairp3y = hairp3y - 0.2;
      hairp4y = hairp4y - 0.2;
    }
  }if (poses.length == 4){
    if (hairp1y &lt; 171){ hairp1y = hairp1y + 0.2; hairp2y = hairp2y + 0.2; hairp3y = hairp3y + 0.2; hairp4y = hairp4y + 0.2; }if (hairp1y &gt; 171){
      hairp1y = hairp1y - 0.2;
      hairp2y = hairp2y - 0.2;
      hairp3y = hairp3y - 0.2;
      hairp4y = hairp4y - 0.2;
    }
  }
 
  if (poses.length &lt; 5){
  curveVertex(hairp1x, hairp1y);
  curveVertex(hairp1x, hairp1y);
  curveVertex(hairp2x, hairp2y);
  curveVertex(hairp3x, hairp3y);
  curveVertex(hairp4x, hairp4y);
  curveVertex(hairp4x, hairp4y);
  endShape();
 
  beginShape();
  curveVertex(hairp1x+20, hairp1y);
  curveVertex(hairp1x+20, hairp1y);
  curveVertex(hairp2x+20, hairp2y);
  curveVertex(hairp3x+20, hairp3y);
  curveVertex(hairp4x+20, hairp4y);
  curveVertex(hairp4x+20, hairp4y);
  endShape();
 
  beginShape();
  curveVertex(hairp1x+40, hairp1y);
  curveVertex(hairp1x+40, hairp1y);
  curveVertex(hairp2x+40, hairp2y);
  curveVertex(hairp3x+40, hairp3y);
  curveVertex(hairp4x+40, hairp4y);
  curveVertex(hairp4x+40, hairp4y);
  endShape();
  }
 
 
}
 
 
 
// A function to draw ellipses over the detected keypoints
function drawKeypoints()  {
  // Loop through all the poses detected
 
  if (poses.length == 1){
    for (let i = 0; i &lt; poses.length; i++) {
      // For each pose detected, loop through all the keypoints
      let pose = poses[i].pose;
      for (let j = 0; j &lt; pose.keypoints.length; j++) {
        // A keypoint is an object describing a body part (like rightArm or leftShoulder)
        let keypoint = pose.keypoints[j];
 
        let noseP = pose.keypoints[0]
        // Only draw an ellipse is the pose probability is bigger than 0.2
        //print(keypoint.position.x);
        push();
        var fx = map(noseP.position.x, 0, width, -1, 1);
        var fy = map(noseP.position.y, 0, height, -1, 1);
        var mx = map(posx, 0, width, -1, 1);
        var my = map(posy, 0, height, -1, 1);
        var dir = atan2((fy - my), (fx - mx));
 
        var amp = 10;
        if (dist(posx, posy, noseP.position.x, noseP.position.y) &lt;= 12){
           amp = dist(posx, posy, noseP.position.x, noseP.position.y);
        }
        translate(posx + amp * cos(dir), posy + amp * sin(dir));
        rotate(dir);
 
        var c = constrain(dist(posx, posy, noseP.position.x, noseP.position.y), 0, 12);
        var distshape = map(c, 0, 30, 1, 0.9);
 
 
        ellipse(0, 0, 5 * distshape, 5);
        pop();
 
      }
    }
  }else{
    ellipse(posx, posy, 5, 5); 
  }
}
 
 
function drawGradient(x, y) {
  var h = 255;
  noStroke();
  if (poses.length == 0){
    if (radius &lt; 106.6){ radius = radius + 0.2; } } if (poses.length == 1){ if (radius &gt; 100){
      radius = radius - 0.2;
    }
    if (radius &lt;= 100){ radius = radius + 0.2; } }if (poses.length ==2){ if (radius &gt; 90){
      radius = radius - 0.2;
    }
    if (radius &lt;= 90){ radius = radius + 0.2; } }if (poses.length ==3){ if (radius &gt; 70){
      radius = radius - 0.2;
    }
    if (radius &lt;= 70){ radius = radius + 0.2; } }if (poses.length ==4){ if (radius &gt; 30){
      radius = radius - 0.2;
    }
    if (radius &lt;= 30){ radius = radius + 0.2; } }if (poses.length ==5){ if (radius &gt; 10){
      radius = radius - 0.2;
    }
    if (radius &lt;= 10){ radius = radius + 0.2; } } for (var r = radius; r &gt; 0; --r) {
    if (poses.length == 0){
      colorR = 250;
      colorG = 210;
    } if (poses.length == 1){
 
      if (colorG &gt; 188){
        colorG -= 0.01;
      }if (colorG &lt; 188){ colorG += 0.01; } } if (poses.length == 2){ if (colorG &gt; 148){
        colorG -= 0.01;
      }
    }
    fill(colorR, colorG-r/3.5, 0);
    ellipse(x, y, r, r);
  }
}
 
 
 
 
/*
 
function setup() {
  createCanvas(710, 400);
  background(255);
  ellipseMode(RADIUS);
}
 
function draw() {
  background(255);
  drawGradient(width/2, height/2);
  push();
  fill(252,238,213)
  ellipse(width/2,height/2,20,20);
  fill(0);
  ellipse(width/2,height/2+10,4,4);
  stroke(4);
  line(width/2,height/2-width/6,width/2,height/2-width/6-20);
  line(width/2-20,height/2-width/6+2,width/2-22,height/2-width/6-18);
  line(width/2+20,height/2-width/6+2,width/2+22,height/2-width/6-18);
  pull();
}
 
function drawGradient(x, y) {
  var h = 255;
  var radius = width/6;
  noStroke();
  for (var r = radius; r &gt; 0; --r) {
    fill(250, 166-r/3.5, 0);
    ellipse(x, y, r, r);
  }
}
*/

chromsan-Body

For this project, I created a visualization of the space where the gaze of two characters in conversation meet. As the characters look at or away from one another, a shape is created between them that grows over the course of a scene.

I used ml5.js and PoseNet to track the characters in the videos and p5.js to draw the shape between them. The field of view for each character is determined by a line drawn from their ear to a point between their eyes and nose. The line is extended outward at two different angles to create a triangle. Both characters' fields of vision are checked for intersecting points and the shape drawn between them is progressively built from these points, layered over and over.

I decided to use scenes from movies mainly because of the one scene from There Will Be Blood where Daniel and Eli engage in this intense standoff, where Daniel talks down to Eli in both the dialogue and his body language. Eli's gaze falls to the ground while Daniel's stays fixed on Eli. This scene worked particularly well, as there is a good amount of change in their gazes throughout, and the shape that emerges between the two is quite interesting. I searched for more scenes with dialogue shot from the side, and came up with a number of them, mostly from P.T. Anderson and Tarantino, the only two directors that seem to use it regularly. I ended up using clips from Kill Bill Vol.I, There Will Be Blood, Boogie Nights, The Big Lebowski, and Inglourious Basterds.

I initially wanted to create one shape between the characters and have their gaze pull it around the screen, but I didn't like the result. So I created a fill for the area where the gazes intersected, and then I created multiple fills that built up to create a complex shape that ends up looking like a brushstroke. There are definitely limitations to this method though; the camera has to be relatively still, there can't be too many characters, and the scene has to be pretty short. Drawing all the shapes on every draw call in addition to running PoseNet in realtime is quite computationally demanding. Even with a good graphics card the video can start to lag; the frames definitely start to drop by the end of some of the longer scenes. Overall, I'm pretty happy with the result, but there is certainly more optimization to be done.

A few more:

Some debug views: 

The code can be found on GitHub.

let poseNet, poses = [];
let video, videoIsPlaying; 
let left = [], right = [], points = [];
let initialAlpha = 100, FOVheight = 150;
let time;
let show = true, debug = true;
 
let vidName = 'milkshake';
let rgb = '#513005';
 
class FOVedge {
 
  constructor(_x1, _y1, _x2, _y2) {
    this.x1 = _x1;
    this.x2 = _x2;
    this.y1 = _y1;
    this.y2 = _y2;
  }
  draw(r, g, b, a){
    stroke(r, g, b, a);
    line(this.x1, this.y1, this.x2, this.y2);
  }
  intersects(l2) {
    let denom = ((l2.y2 - l2.y1) * (this.x2 - this.x1) - (l2.x2 - l2.x1) * (this.y2 - this.y1));
    let ua = ((l2.x2 - l2.x1) * (this.y1 - l2.y1) - (l2.y2 - l2.y1) * (this.x1 - l2.x1)) / denom;
    let ub = ((this.x2 - this.x1) * (this.y1 - l2.y1) - (this.y2 - this.y1) * (this.x1 - l2.x1)) / denom;
    if (ua &lt; 0 || ua &gt; 1 || ub &lt; 0 || ub &gt; 1) return false;
    let x = this.x1 + ua * (this.x2 - this.x1);
    let y = this.y1 + ua * (this.y2 - this.y1);
    return {x, y};
  }
}
 
class FOV {
  constructor(x1, y1, x2, y2) {
    this.s1 = new FOVedge(x1, y1, x2, y2 - FOVheight);
    this.s2 = new FOVedge(x1, y1, x2, y2 + FOVheight);
    this.s3 = new FOVedge(x2, y2 + FOVheight, x2, y2 - FOVheight);
    this.col = color(191, 191, 191, initialAlpha); 
    this.show = true;
    // right = 1, left = -1
    if (x2 &gt; x1) {
      this.direction = 1;
    } else {
      this.direction = -1
    }
  }
  checkForIntersections(fov) {
    let intersections = [];
    for (let j = 1; j &lt; 3; j++){
      let sattr = 's' + j.toString();
      for (let i = 1; i &lt; 3; i++) {
        let attr = 's' + i.toString();
        let ints = this[sattr].intersects(fov[attr]);
        if (ints != false) {
          intersections.push(ints);
        }
      }
    }
    if (intersections.length == 2){
      intersections.push({x: this.s1.x1, y: this.s1.y1});
    }
    return intersections;
  }
  fade(){
    this.col.levels[3] = this.col.levels[3] - 3;
    if (this.col.levels[3] &lt; 0) {
      this.show = false;
    }
  } 
  draw() {
    this.s1.draw(this.col.levels[0], this.col.levels[1], this.col.levels[2], this.col.levels[3]);
    this.s2.draw(this.col.levels[0], this.col.levels[1], this.col.levels[2], this.col.levels[3]);
    this.s3.draw(this.col.levels[0], this.col.levels[1], this.col.levels[2], this.col.levels[3]);
  }
}
 
function setup() {
  videoIsPlaying = false; 
  createCanvas(1280, 720, P2D);
  //createCanvas(1920, 1080);
  video = createVideo( vidName + '.mp4', vidLoad);
  video.size(width, height);
  poseNet = ml5.poseNet(video, modelReady);
  poseNet.on('pose', function(results) {
    poses = results;
  });
  video.hide();
}
 
function modelReady() {
  select('#status').html('Model Loaded');
}
 
function mousePressed(){
  vidLoad();
}
 
function draw() {
  if (show) {
    image(video, 0, 0, width, height);
  } else {
    background(214, 214, 214);
  }
 
  drawKeypoints();
}
 
function drawKeypoints()  {
 
  for (let i = 0; i &lt; poses.length; i++) {
 
    let pose = poses[i].pose;
    for (let j = 0; j &lt; 5; j++) { let keypoint = pose.keypoints[j]; if ((j == 3 || j == 4) &amp;&amp; keypoint.score &gt; 0.7) { // left or right ear
        // calclulate average x between nose and eye
        let earX = 0, earY = 0;
        if (pose.keypoints[2].score &gt; 0.7){
          earX = pose.keypoints[2].position.x;
          earY = pose.keypoints[2].position.y;
        } else {
          earX = pose.keypoints[1].position.x;
          earY = pose.keypoints[1].position.y;
        }
         let x1 = keypoint.position.x
         let y1 = keypoint.position.y
         let x2 = earX;
         let y2 = earY;
         //let x2 = (earX + pose.keypoints[0].position.x) / 2;
         //let y2 = (earY + pose.keypoints[0].position.y) / 2;
         let length = Math.sqrt(Math.pow(x1 - x2, 2) + pow(y1 - y2 , 2));
         let newX = x2 + (x2 - x1) / length * 1200;
         let newY = y2 + (y2 - y1) / length * 1200;
 
         let look = new FOV(x2, y2, newX, newY);
 
         if (look.direction == -1) {
          let lastR = right.pop();
          if (lastR != undefined){
            let ints = look.checkForIntersections(lastR)
            if (ints.length &gt;= 3 &amp;&amp; show) {
              points.push(ints);
            }
            right.push(lastR);
          }
          left.push(look)
         } else {
          let lastL = left.pop();
          if (lastL != undefined) {
            let ints = look.checkForIntersections(lastL)
            if (ints.length &gt;= 3 &amp;&amp; show) {
              points.push(ints);
            }
            left.push(lastL);
          }
          look.col =  color(56, 56, 56, initialAlpha); 
          right.push(look)
         }
      }
      let col = color(rgb);
      col.levels[3] = 2;
      fill(col.levels[0], col.levels[1], col.levels[2], col.levels[3]);
      noStroke();
      if (!debug){
      for (let i = 0; i &lt; points.length; i++) {
        beginShape();
        for (let j = 0; j &lt; points[i].length; j++) {
          vertex(points[i][j].x, points[i][j].y);
        }
        endShape(CLOSE)
      }
    }
      for (let i = 0; i &lt; left.length; i++) {
        if (debug){left[i].draw();}
        left[i].fade();
        if (!left[i].show) {
          left.splice(i, 1);
        }
      }
      for (let i = 0; i &lt; right.length; i++) {
        if (debug){right[i].draw();}
        right[i].fade();
        if (!right[i].show) {
          right.splice(i, 1);
        }
      }
    }
  }
}
 
function vidLoad() {
  time = video.duration();
  video.stop();
  video.loop();
  videoIsPlaying = true;
  if (!debug) {
  setTimeout(function(){ 
    show = false;
    video.volume(0);
    video.hide();
   }, time * 1000);
}
}
function keyPressed(){
  if (videoIsPlaying) {
    video.pause();
    videoIsPlaying = false;
  } else {
    video.loop();
    videoIsPlaying = true;
  }
}

breep-Body

 

For this piece the motion definitely came first. I most experimented with ways of conveying the body through lines while getting to know Mocap in Processing, the video underneath is one of the more refined outcomes of my experimentation. With my series of experimentations in place, I thought through each and wanted to build a form of narrative in which this digital body wills itself into existence from an initial form as a plane. From this I came to the final piece, which moves toward the viewer just as the walking motion does, urging itself onwards into existence as the viewer scrolls through.

I am mildly pleased with the outcome, but not happy. I feel that greater blending between the states with more steps would make this a more cohesive transition.

 

 

 
// Based off of Template provided on 60212 Fall 2018 Website
 
 
// Renders a BVH file with Processing v3.4
// Note: mouseX controls the camera.
// Based off of com.rhizomatiks.bvh
// Golan Levin, September 2018
 
 
 
PBvh myBrekelBvh;
PBvh secondBrekelBvh; 
PBvh thirdBrekelBvh;
PBvh fourthBrekelBvh; 
PBvh fifthBrekelBvh; 
boolean bDrawMeat = true; 
boolean bDrawLineConnectingHands = false; 
boolean bDrawCircleAtOrigin = false; 
boolean bDrawLineToBoneInScreenspace = false; 
float boneScreenX; 
float boneScreenY; 
int xCamera = 500; 
int yCamera = 500; 
int zCamera = 500; 
int distance = 3200; 
int distanceFinal = 4000; 
int distanceWalkout = 4400; 
 
//------------------------------------------------
void setup() {
  size( 1280, 720, P3D );
  // Load a BVH file recorded with a Kinect v2, made in Brekel Pro Body v2.
  myBrekelBvh = new PBvh( loadStrings("walk_bold.bvh" ) );
 
  secondBrekelBvh = new PBvh( loadStrings("walk-cycle.bvh") ); 
 
}
 
 
//------------------------------------------------
void draw() {
  lights() ;
  background(0, 0, 0);
 
  pushMatrix(); 
  setMyCamera();        // Position the camera. See code below.
  //drawMyGround();       // Draw the ground. See code below.
  updateAndDrawBody();  // Update and render the BVH file. See code below.
  popMatrix(); 
 
  drawHelpInfo();
  drawLineToHead();
 
 
}
 
void keyPressed() {
  switch (key) {
  case 'M': 
  case 'm': 
    bDrawMeat = !bDrawMeat; 
    break;
  case 'L':
  case 'l': 
    bDrawLineConnectingHands = !bDrawLineConnectingHands;
    break;
  case 'O': 
  case 'o': 
    bDrawCircleAtOrigin = !bDrawCircleAtOrigin; 
    break;
  case 'H': 
  case 'h':
    bDrawLineToBoneInScreenspace = !bDrawLineToBoneInScreenspace; 
    break;
 
  case 'w': 
    yCamera += 50; 
    break; 
  case 's':
    yCamera -= 50; 
    break; 
  case 'd':
    xCamera += 50; 
    break; 
  case 'a':
    xCamera -= 50; 
    break; 
 
  case 'f':
    zCamera += 50; 
    break; 
  case 'g':
    zCamera -= 50; 
    break; 
 
  }
}
 
 
//------------------------------------------------
void updateAndDrawBody() {
 
  pushMatrix(); 
  translate(width/2, height/2, 0); // position the body in space
  scale(-1, -1, 1); // correct for the coordinate system orientation
  myBrekelBvh.update(millis()); // update the BVH playback
 
  pushMatrix();
  stroke(255, 0, 0); 
  //myBrekelBvh.draw();        // one way to draw the BVH file (see PBvh.pde)
  popMatrix(); 
 
 
  if (bDrawMeat) {
    myBrekelBvh.drawBones(); // a different way to draw the BVH file
  }
 
  if (bDrawLineConnectingHands) {
    drawLineConnectingHands();
  }
 
  if (bDrawCircleAtOrigin) {
    drawCircleAtOrigin();
  }
 
  if (bDrawLineToBoneInScreenspace) {
    myBrekelBvh.calculateScreenspaceLocationOfBone("Head");
  }
 
  popMatrix();
 
 
  pushMatrix(); 
  translate(width/2, height/2, 0); // position the body in space
  scale(-1, -1, 1); // correct for the coordinate system orientation
  myBrekelBvh.update(millis()); // update the BVH playback
 
  pushMatrix();
  stroke(255, 0, 0); 
  //myBrekelBvh.draw();        // one way to draw the BVH file (see PBvh.pde)
  popMatrix(); 
 
 
  if (bDrawMeat) {
    secondBrekelBvh.drawBones(); // a different way to draw the BVH file
  }
 
  if (bDrawLineConnectingHands) {
    drawLineConnectingHands();
  }
 
  if (bDrawCircleAtOrigin) {
    drawCircleAtOrigin();
  }
 
  if (bDrawLineToBoneInScreenspace) {
    secondBrekelBvh.calculateScreenspaceLocationOfBone("Head");
  }
 
  popMatrix();
 
}
 
//------------------------------------------------
 
void drawHelpInfo() {
  fill(255); 
  float ty = 20; 
  float dy = 15; 
  //text("BVH Loader for Processing 3.4", 20, ty+=dy);
  //text("Displays files recorded with Kinect v2 + Brekel Pro Body 2", 20, ty+=dy);
 
  //text("", 20, ty+=dy);
  //text("Press 'M' to toggle meat", 20, ty+=dy);
  //text("Press 'L' to draw line connecting hands", 20, ty+=dy);
  //text("Press 'O' to draw circle at Origin", 20, ty+=dy);
  //text("Press 'H' to draw line from mouse to Head", 20, ty+=dy);
}
 
 
//------------------------------------------------
 
void drawLineToHead() {
  if (bDrawLineToBoneInScreenspace) {
    stroke(0, 255, 255); 
    strokeWeight(3); 
    line (mouseX, mouseY, boneScreenX, boneScreenY);
  }
}
 
 
//------------------------------------------------
 
void drawCircleAtOrigin() {
  fill(255); 
  stroke(255, 0, 0); 
  strokeWeight(4); 
  ellipse(0, 0, 20, 20);
}
 
 
//------------------------------------------------
void drawLineConnectingHands() {
  // This example code shows how to reach into the skeleton 
  // in order to get specific joint locations (in 3D)
 
  PVector[] boneLocationsFirst = new PVector[19]; 
  PVector[] boneLocationsSecond = new PVector[19]; 
 
  for (int i = 0; i < 19; i ++){
    int currentIndex = i; 
    BvhBone currentBoneFirst = myBrekelBvh.parser.getBones().get(currentIndex); 
    PVector currentBonePosFirst = currentBoneFirst.absPos;
 
    BvhBone currentBoneSecond = secondBrekelBvh.parser.getBones().get(currentIndex); 
    PVector currentBonePosSecond = currentBoneSecond.absPos;
 
    boneLocationsFirst[i] = currentBonePosFirst;
    boneLocationsSecond[i] = currentBonePosSecond; 
  }
 
 
  for (int j = 0; j < 19; j ++){
      for (int z = 0; z < 19; z ++){ 
      PVector currentPosFirst = boneLocationsFirst[j]; 
      PVector secondPosFirst = boneLocationsFirst[z]; 
 
      PVector currentPosSecond = boneLocationsSecond[j];
      PVector secondPosSecond = boneLocationsSecond[z]; 
 
      stroke(255, 255, 255); 
      strokeWeight(1); 
 
      // Body Lines 
 
      pushMatrix(); 
 
      if (j == 1) {
        for (int a = 1; a < 4; a ++){
        line( 7 * boneLocationsFirst[a].x, 7 * boneLocationsFirst[a].y, boneLocationsFirst[a].z + distance, 
             7 * boneLocationsFirst[a+1].x, 7 *boneLocationsFirst[a+1].y, boneLocationsFirst[a+1].z + distance);
        }
      } 
 
      if (j == 5) { 
        line( 7 * boneLocationsFirst[3].x, 7 * boneLocationsFirst[3].y, boneLocationsFirst[3].z + distance, 
             7 * boneLocationsFirst[5].x, 7 * boneLocationsFirst[5].y, boneLocationsFirst[5].z + distance);
 
        for (int b = 5; b < 8; b ++){
        line( 7 * boneLocationsFirst[b].x, 7 * boneLocationsFirst[b].y, boneLocationsFirst[b].z + distance, 
             7 * boneLocationsFirst[b+1].x, 7 * boneLocationsFirst[b+1].y, boneLocationsFirst[b+1].z + distance);
        }
      }
 
      if (j == 9) { 
        line( 7 * boneLocationsFirst[3].x, 7 * boneLocationsFirst[3].y, boneLocationsFirst[3].z + distance, 
             7 * boneLocationsFirst[9].x, 7 *boneLocationsFirst[9].y, boneLocationsFirst[9].z + distance);
 
        for (int c = 9; c < 12; c ++){
        line( 7 * boneLocationsFirst[c].x, 7 * boneLocationsFirst[c].y, boneLocationsFirst[c].z + distance, 
             7 * boneLocationsFirst[c+1].x, 7 *boneLocationsFirst[c+1].y, boneLocationsFirst[c+1].z + distance);
        }
      }
 
      if (j == 13) { 
        line( 7 * boneLocationsFirst[1].x, 7 * boneLocationsFirst[1].y, boneLocationsFirst[1].z + distance, 
             7 * boneLocationsFirst[13].x, 7 *boneLocationsFirst[13].y, boneLocationsFirst[13].z + distance);
 
        for (int d = 13; d < 15; d ++){
        line( 7 * boneLocationsFirst[d].x, 7 * boneLocationsFirst[d].y, boneLocationsFirst[d].z + distance, 
             7 * boneLocationsFirst[d+1].x, 7 *boneLocationsFirst[d+1].y, boneLocationsFirst[d+1].z + distance);
        }
      }
 
      if (j == 16) { 
        line( 7 * boneLocationsFirst[1].x, 7 * boneLocationsFirst[1].y, boneLocationsFirst[1].z + distance, 
             7 * boneLocationsFirst[16].x, 7 *boneLocationsFirst[16].y, boneLocationsFirst[16].z + distance);
 
        for (int e = 16; e < 18; e ++){
        line( 7 * boneLocationsFirst[e].x, 7 * boneLocationsFirst[e].y, boneLocationsFirst[e].z + distance, 
             7 * boneLocationsFirst[e+1].x, 7 *boneLocationsFirst[e+1].y, boneLocationsFirst[e+1].z + distance);
        }
      }
 
      /* translate(0, 0, distance); 
      ellipse(7 * boneLocationsFirst[4].x, 7 * boneLocationsFirst[4].y + 30, 20, 20); */ 
 
      popMatrix(); 
 
      ////----------------------
 
      pushMatrix(); 
 
      if (j == 1) {
        for (int a = 1; a < 4; a ++){
        line( 8 * boneLocationsFirst[a].x, 8 * boneLocationsFirst[a].y, boneLocationsFirst[a].z + distanceFinal, 
             8 * boneLocationsFirst[a+1].x, 8 *boneLocationsFirst[a+1].y, boneLocationsFirst[a+1].z + distanceFinal);
        }
      } 
 
      if (j == 5) { 
        line( 8 * boneLocationsFirst[3].x, 8 * boneLocationsFirst[3].y, boneLocationsFirst[3].z + distanceFinal, 
             8 * boneLocationsFirst[5].x, 8 * boneLocationsFirst[5].y, boneLocationsFirst[5].z + distanceFinal);
 
        for (int b = 5; b < 8; b ++){
        line( 8 * boneLocationsFirst[b].x, 8 * boneLocationsFirst[b].y, boneLocationsFirst[b].z + distanceFinal, 
             8 * boneLocationsFirst[b+1].x, 8 * boneLocationsFirst[b+1].y, boneLocationsFirst[b+1].z + distanceFinal);
        }
      }
 
      if (j == 9) { 
        line( 8 * boneLocationsFirst[3].x, 8 * boneLocationsFirst[3].y, boneLocationsFirst[3].z + distanceFinal, 
             8 * boneLocationsFirst[9].x, 8 *boneLocationsFirst[9].y, boneLocationsFirst[9].z + distanceFinal);
 
        for (int c = 9; c < 12; c ++){
        line( 8 * boneLocationsFirst[c].x, 8 * boneLocationsFirst[c].y, boneLocationsFirst[c].z + distanceFinal, 
             8 * boneLocationsFirst[c+1].x, 8 *boneLocationsFirst[c+1].y, boneLocationsFirst[c+1].z + distanceFinal);
        }
      }
 
      if (j == 13) { 
        line( 8 * boneLocationsFirst[1].x, 8 * boneLocationsFirst[1].y, boneLocationsFirst[1].z + distanceFinal, 
             8 * boneLocationsFirst[13].x, 8 *boneLocationsFirst[13].y, boneLocationsFirst[13].z + distanceFinal);
 
        for (int d = 13; d < 15; d ++){
        line( 8 * boneLocationsFirst[d].x, 8 * boneLocationsFirst[d].y, boneLocationsFirst[d].z + distanceFinal, 
             8 * boneLocationsFirst[d+1].x, 8 *boneLocationsFirst[d+1].y, boneLocationsFirst[d+1].z + distanceFinal);
        }
      }
 
      if (j == 16) { 
        line( 8 * boneLocationsFirst[1].x, 8 * boneLocationsFirst[1].y, boneLocationsFirst[1].z + distanceFinal, 
             8 * boneLocationsFirst[16].x, 8 *boneLocationsFirst[16].y, boneLocationsFirst[16].z + distanceFinal);
 
        for (int e = 16; e < 18; e ++){
        line( 8 * boneLocationsFirst[e].x, 8 * boneLocationsFirst[e].y, boneLocationsFirst[e].z + distanceFinal, 
             8 * boneLocationsFirst[e+1].x, 8 *boneLocationsFirst[e+1].y, boneLocationsFirst[e+1].z + distanceFinal);
        }
      }
 
      /* translate(0, 0, distance); 
      ellipse(7 * boneLocationsFirst[4].x, 7 * boneLocationsFirst[4].y + 30, 20, 20); */ 
 
      popMatrix(); 
 
 
      // Points of body to backside origin  
 
      pushMatrix(); 
           translate(0, 0, 2400); 
            line( 6 * currentPosFirst.x, 6 * currentPosFirst.y, currentPosFirst.z, 
                  0, 0 , -400); 
            popMatrix(); 
 
 
       // Points of body to backside origin  
 
      pushMatrix(); 
           translate(0, 0, 3200); 
            line( 6 * currentPosFirst.x, 6 * currentPosFirst.y, currentPosFirst.z, 
                  0, 0 , -400); 
            popMatrix(); 
 
 
       //BaseLine
       pushMatrix(); 
          stroke(255); 
          line(
            currentPosFirst.x + 10000, currentPosFirst.y, currentPosFirst.z - 400,
            currentPosSecond.x - 10000, currentPosSecond.y, currentPosSecond.z - 400); 
          popMatrix();
 
 
       //Lines extrapolating to one direction
          pushMatrix(); 
          stroke(255); 
          line(
            currentPosFirst.x + 10, currentPosFirst.y, currentPosFirst.z,
            currentPosFirst.x - 10, currentPosFirst.y, currentPosFirst.z); 
          popMatrix();
 
 
       /* //Lines extrapolating to one direction
          pushMatrix(); 
          stroke(255); 
          line(
            currentPosFirst.x + 50, currentPosFirst.y, currentPosFirst.z + 100,
            currentPosFirst.x - 50, currentPosFirst.y, currentPosFirst.z + 100); 
          popMatrix();
 
       //Lines extrapolating to one direction
          pushMatrix(); 
          stroke(255); 
          line(
            currentPosFirst.x + 25, currentPosFirst.y, currentPosFirst.z + 200,
            currentPosFirst.x - 25, currentPosFirst.y, currentPosFirst.z + 200); 
          popMatrix();
 
 
      //Lines extrapolating to one direction
          pushMatrix(); 
          stroke(255); 
          line(
            currentPosFirst.x + 10, currentPosFirst.y, currentPosFirst.z + 300,
            currentPosFirst.x - 10, currentPosFirst.y, currentPosFirst.z + 300); 
          popMatrix();*/ 
 
 
      //Lines in First Body
 
      line(
        2 * currentPosFirst.x, 2 * currentPosFirst.y, currentPosFirst.z + 400,
        2 * secondPosFirst.z, 2 * secondPosFirst.y, secondPosFirst.z + 400);
 
 
      for (int k = 0; k < 8; k ++){
 
          //Tutu
 
          float angle = map(k, 0, 8, 0, TWO_PI); 
 
          /*if (k % 2 == 0){
          stroke(255, 0, 0);}
 
          else{
          stroke(0, 0, 255);}*/
 
          stroke(255); 
 
          line(
            3 * currentPosFirst.x, 3 * currentPosFirst.y, currentPosFirst.z + 800,
            100 * cos(angle), 50, 100 * sin(angle) + 800); 
 
          /*line(
            5 * currentPosSecond.x, 5 * currentPosSecond.y, currentPosSecond.z, 
            100 * cos(angle), 50, 100 * sin(angle));*/
 
          pushMatrix(); 
 
          /*if (k % 2 == 0){
          stroke(0, 255, 0);}
 
          else{
          stroke(255, 0, 0);}*/ 
 
          stroke(255); 
 
          line(
            4 * currentPosFirst.x, 4 * currentPosFirst.y, currentPosFirst.z + 1600,
            100 * sin(angle), (100 * cos(angle)), 100 * cos(angle) + 1200); 
 
          popMatrix(); 
 
 
 
          /*pushMatrix(); 
          translate(0, 0, 1200); 
          stroke(255, 0, 0); 
          ellipse(4 * currentPosFirst.x, 4 * currentPosFirst.y, 10, 10); 
          popMatrix();*/ 
 
          /*pushMatrix(); 
          translate(0, 0, 2000); 
          stroke(255); 
          ellipse(5 * currentPosFirst.x, 5 * currentPosFirst.y, 5, 5); 
          popMatrix(); */
 
          /*pushMatrix(); 
          translate(0, 0, 2000); 
          stroke(255, 0, 0); 
          ellipse(6 * currentPosFirst.x, 6 * currentPosFirst.y, 50, 50); 
          popMatrix(); 
 
          pushMatrix(); 
          translate(0, 0, 2000); 
          stroke(255, 0, 0); 
          ellipse(6 * currentPosFirst.x, 6 * currentPosFirst.y, 100, 100); 
          popMatrix(); */ 
 
 
          /*pushMatrix(); 
          stroke(255); 
          line(
            4 * currentPosFirst.x, 4 * currentPosFirst.y, currentPosFirst.z + 600,
            5 * currentPosFirst.x, 5 * currentPosFirst.y, currentPosFirst.z + 1000); 
          translate(0,0, 10000); 
          ellipse(currentPosFirst.x, currentPosFirst.y, 50, 50); 
          popMatrix(); */ 
      } 
      }
      }
  }
 
 
          /*pushMatrix(); 
 
          if (k % 2 == 0){
          stroke(0, 255, 0);}
 
          else{
          stroke(255, 0, 0);}
 
          line(
            currentPosFirst.x, currentPosFirst.y, currentPosFirst.z ,
            100 * sin(angle), (100 * sin(angle)) + (100 * cos(angle)), 100 * cos(angle)); 
 
          popMatrix();*/ 
 
 
 
          //Lines extrapolating to one direction to Origin
          /*pushMatrix(); 
          if (k == 0){
            stroke(0, 255, 0);}
          else if (k == 1){
            stroke(255, 0, 0);}
          else {
            stroke(0, 0, 255);}
          translate(-(60*k), 0, 0); 
          line(
            currentPosSecond.x + (60 * k), currentPosSecond.y, currentPosSecond.z,
            0, 200, 200); 
          popMatrix(); */
 
 
 
 
//------------------------------------------------
void setMyCamera() {
 
 
  // Adjust the position of the camera
  float eyeX = 650;          // x-coordinate for the eye
  float eyeY = yCamera; //height/2.0f - 200;   // y-coordinate for the eye
  float eyeZ = mouseY * 8;             // z-coordinate for the eye
  float centerX = width/2.0f;   // x-coordinate for the center of the scene
  float centerY = height/2.0f;  // y-coordinate for the center of the scene
  float centerZ = -400;         // z-coordinate for the center of the scene
  float upX = 0;                // usually 0.0, 1.0, or -1.0
  float upY = 1;                // usually 0.0, 1.0, or -1.0
  float upZ = 0;                // usually 0.0, 1.0, or -1.0
 
  camera(eyeX, eyeY, eyeZ, centerX, centerY, centerZ, upX, upY, upZ);
 
}
 
//------------------------------------------------
void drawMyGround() {
  // Draw a grid in the center of the ground 
  pushMatrix(); 
  translate(width/2, height/2, 0); // position the body in space
  scale(-1, -1, 1);
 
  stroke(100);
  strokeWeight(1); 
 
  float gridSize = 400; 
  int nGridDivisions = 10; 
 
  for (int col=0; col<=nGridDivisions; col++) {
    float x = map(col, 0, nGridDivisions, -gridSize, gridSize);
    line (x, 0, -gridSize, x, 0, gridSize);
  }
  for (int row=0; row<=nGridDivisions; row++) {
    float z = map(row, 0, nGridDivisions, -gridSize, gridSize); 
    line (-gridSize, 0, z, gridSize, 0, z);
  }
 
  popMatrix();
}

 

 

weirdie-Body

Interactive Shadow Box

I knew for this project that I wanted to create something that specifically responded to the face. I was interested in how the movement of the face could be translated to control something non-human, such as a butterfly. For this reason I chose to use FaceOSC, as I wanted to utilize the gestures such as head tilt and eye openness to control movement. The goal was to create a sort of interactive shadow box, where the image appears still until it detects a face. I would be interested in creating a whole display set of them in the future with other insects and other control gestures. The wings were drawn in Photoshop, created as two separate images which are rotated about the y-axis based on how open your eyes are.

After being frustrated with how jittery the movement appeared, I utilized a circular buffer that takes a running average of previous points to translate to the actual movement of the butterfly. This helped significantly, though it could certainly be refined further. Additionally, I would like to add a cast shadow from the butterfly to add depth.

GIF of some normal blinking:

GIF with really aggressive squinting:

Still image of just the shadow box:

import oscP5.*;
OscP5 oscP5;
 
int     found; // global variable, indicates if a face is found
PVector poseOrientation = new PVector(); // stores an (x,y,z)
float leftOpen;
float rightOpen;
PImage wingRight;
PImage wingLeft;
PImage bckgrd;
CircularBuffer leftBuff = new CircularBuffer(10);
CircularBuffer rightBuff = new CircularBuffer(10);
 
 
//----------------------------------
void setup() {
  size(800, 800, OPENGL);
  oscP5 = new OscP5(this, 8338);
  oscP5.plug(this, "found", "/found");
  oscP5.plug(this, "poseOrientation", "/pose/orientation");
  oscP5.plug(this, "leftOpen", "/gesture/eye/left");
  oscP5.plug(this, "rightOpen", "/gesture/eye/right");
  oscP5.plug(this, "leftBrow", "/gesture/eyebrow/left");
  oscP5.plug(this, "rightBrow", "/gesture/eyebrow/right");
 
  wingRight = loadImage("wingr.png");
  wingLeft = loadImage("wingl.png");
  bckgrd = loadImage("background.png");
}
 
//----------------------------------
void draw() {
  background (214, 205, 197);
  background(178, 163, 149);
  image(bckgrd, 0, 0, width, height);
 
  noFill();
  float scl = 250;
 
  if (found != 0) {
    pushMatrix(); 
    translate (width/2, height/2, 0);
    rotateZ (poseOrientation.z);
    float rightRotate = filter(rightBuff);
    rotateY (constrain(map(rightRotate, 2.7, 3.7, -PI/2, PI/6), -PI/2+0.1, -0.05)); 
    image(wingRight, 0, -200, scl, 1.4*scl);
    popMatrix();
 
    pushMatrix();
    translate (width/2, height/2, 0);
    rotateZ (poseOrientation.z);
    float leftRotate = filter(leftBuff);
    rotateY (constrain(map(leftRotate, 2.7, 3.7, PI/2, -PI/6), 0.05, PI/2-0.1));
    image(wingLeft, 0, -200, -scl, 1.4*scl);
    popMatrix();
  }
  else
  {
    pushMatrix();
    translate (width/2, height/2, 0);
    image(wingRight, 0, -200, scl, 1.4*scl);
    image(wingLeft, 0, -200, -scl, 1.4*scl);
    popMatrix();
  }
 
  fill(37, 34, 27);
  noStroke();
  rect(0, 0, width, 30);
  rect(0, 0, 30, height);
  rect(0, height-30, width, 30);
  rect(width-30, 0, 30, height);
}
 
//----------------------------------
// Event handlers for receiving FaceOSC data
public void found (int i) { found = i; }
public void poseOrientation(float x, float y, float z) {
  poseOrientation.set(x, y, z);
}
public void leftOpen (float i) {leftOpen = i; leftBuff.store(i);}
public void rightOpen (float i) {rightOpen = i; rightBuff.store(i);}
 
//----------------------------------
// Event handlers for receiving FaceOSC data
public void found (int i) { found = i; }
public void poseOrientation(float x, float y, float z) {
  poseOrientation.set(x, y, z);
}
public void leftOpen (float i) {leftOpen = i; leftBuff.store(i);}
public void rightOpen (float i) {rightOpen = i; rightBuff.store(i);}
public float filter (CircularBuffer buff)
{
  float filt = 0;
  for (int i = 0; i < buff.data.length; i++)
  {
    filt = filt + buff.data[i];
  }
  return filt/buff.data.length;
}
 
//----------------------------------
//CIRCULAR BUFFER CLASS -- keeps past datapoints and calculates average
//to help with smoothing the movement
public class CircularBuffer {
    public float[] data = null;
 
    private int capacity  = 0;
    private int writePos  = 0;
    private int available = 0;
 
    public CircularBuffer(int capacity) {
        this.capacity = capacity;
        this.data = new float[capacity];
    }
 
    public void reset() {
        this.writePos = 0;
        this.available = 0;
    }
 
    public int capacity() { return this.capacity; }
    public int available(){ return this.available; }
 
    public int remainingCapacity() {
        return this.capacity - this.available;
    }
 
    public void store(float element){
 
 
            if(writePos >= capacity){
                writePos = 0;
            }
            data[writePos] = element;
            writePos++;
            available++;
    }
}

yalbert-body

I really struggled to come up with a compelling idea for this project. Initially, I wanted to do something with hands using openpose. Unfortunately, it was too slow on my computer to use live.

After unsuccessfully attempting openpose, I shifted my focus to FaceOSC. I explore a couple different options, including a rolling ping pong ball, before settling on this. Initially inspired by a project Marisa Lu made in the class, I wanted to create a drawing tool where the controller is one's face. Early functionality brainstorms included spewing particles out of one's mouth and 'licking' such particles to move them around. Unfortunately, faceOSC's tongue detection system is not great so I had to shift directions.

Thinking back to the image processing project for 15-104, I thought it would be fun if the particles 'revealed' the user's face. Overall, I'm happy with the elegant simplicity of the piece. I like Char's observation when she described it as a 'modern age peephole'.However, I'm still working on a debug mode in case faceOSC doesn't read the mouth correctly.

 

ParticleSystem ps;
import processing.video.*;
 
Capture cam;
 
import oscP5.*;
OscP5 oscP5;
int found;
float[] rawArray;
Mouth mouth;
int particleSize = 3;
int numParticles = 20;
 
void setup() {
  size(640, 480);
  ps = new ParticleSystem(new PVector(width/2, 50));
  frameRate(30);
 
  setupOSC();
  setupCam();
 
  mouth = new Mouth();
}
 
void setupOSC(){
  rawArray = new float[132]; 
  oscP5 = new OscP5(this, 8338);
  oscP5.plug(this, "found", "/found");
  oscP5.plug(this, "rawData", "/raw");
}
 
void setupCam(){
  String[] cameras = Capture.list();
 
  if (cameras.length == 0) {
    println("There are no cameras available for capture.");
    exit();
  } else {
    println("Available cameras:");
    for (int i = 0; i &lt; cameras.length; i++) {
      println(cameras[i]);
    }
 
    // The camera can be initialized directly using an 
    // element from the array returned by list():
    cam = new Capture(this, cameras[0]);
    cam.start();     
  } 
}
 
void draw() {
  if (cam.available() == true) {
    cam.read();
  }
  translate(width, 0);
  scale(-1, 1);
  background(0);  
  mouth.update();
  //mouth.drawDebug();
  ps.origin =new  PVector(mouth.x, mouth.y);
  if(mouth.isOpen || mouth.isBlowing){
      ps.addParticle();
  }
  ps.run();
  //image(cam, 0, 0);
}
 
void drawFacePoints() {
  int nData = rawArray.length;
  for (int val=0; val&lt;nData; val+=2) {
      fill(100, 100, 100);
      ellipse(rawArray[val], rawArray[val+1], 11, 11);
  }
}
 
 
// A class to describe a group of Particles
// An ArrayList is used to manage the list of Particles 
 
class ParticleSystem {
  ArrayList particles;
  PVector origin;
 
  ParticleSystem(PVector position) {
    origin = position.copy();
    particles = new ArrayList();
  }
 
  void addParticle() {
    for(int i = 0; i &lt; numParticles; i++){ particles.add(new Particle(origin)); } } void run() { for (int i = particles.size()-1; i &gt;= 0; i--) {
      Particle p = particles.get(i);
      p.run();
      if (p.isDead()) {
        particles.remove(i);
      }
    }
  }
}
class Mouth{
 boolean isOpen;
 boolean isBlowing;
 float h;
 float w;
 float x;
 float y;
 float xv;
 float yv;
 
 void update(){
   PVector leftEdge = new PVector(rawArray[96], rawArray[97]);
   PVector rightEdge = new PVector(rawArray[108], rawArray[109]);
   PVector upperLipTop = new PVector(rawArray[102], rawArray[103]);
   PVector upperLipBottom = new PVector(rawArray[122], rawArray[123]);
   PVector lowerLipTop = new PVector(rawArray[128], rawArray[129]);
   PVector lowerLipBottom = new PVector(rawArray[114], rawArray[115]);
 
   float lastx = x;
   float lasty = y;
   w = rightEdge.x - leftEdge.x;
   x = (rightEdge.x - leftEdge.x)/2 + leftEdge.x;
   y = (lowerLipBottom.y - upperLipTop.y)/2 + upperLipTop.y;
   h = lowerLipBottom.y - upperLipTop.y;
   float distOpen = lowerLipTop.y - upperLipBottom.y;
   float avgLipThickness = ((lowerLipBottom.y - lowerLipTop.y) + 
                           (upperLipBottom.y - upperLipTop.y))/2;
   if(distOpen &gt; avgLipThickness){ isOpen = true;}
   else { isOpen = false;}
 
   if(w/h &lt;= 1.5){ isBlowing = true;}
   else { isBlowing = false;}
 
   xv = x - lastx;
   yv = y - lasty;
 
 }
 void drawDebug(){
   if(isOpen || mouth.isBlowing){
        strokeWeight(5);
       stroke(255, 255, 255, 150);
       noFill();
       ellipse(x, y, w, h);
   }
 }
}
 
 
// A simple Particle class
 
class Particle {
  PVector position;
  PVector velocity;
  PVector acceleration;
  float lifespan;
 
  Particle(PVector l) {
    acceleration = new PVector(0, 0.00);
    velocity = new PVector(random(-1, 1), random(-2, 0));
    position = l.copy();
    lifespan = 255.0;
  }
 
  void run() {
    update();
    display();
  }
 
  // Method to update position
  void update() {
    velocity.add(acceleration);
    position.add(velocity);
    //lifespan -= 1.0;
    velocity.x = velocity.x *.99;
    velocity.y = velocity.y *.99;
  }
 
  // Method to display
  void display() {
    //stroke(255, lifespan);
    //image(cam, 0, 0);
    float[] col = getColor(position.x, position.y);
    fill(col[0], col[1], col[2]);
    noStroke();
    ellipse(position.x, position.y, particleSize,particleSize);
  }
 
  // Is the particle still useful?
  boolean isDead() {
    if (lifespan &lt; 0.0) { return true; } else { return false; } } } public float[] getColor(float x, float y){ cam.loadPixels(); int index = int(y)*width +int(x); float[] col = {0, 0, 0}; if(index &gt; 0 &amp;&amp; index &lt; cam.pixels.length){
    col[0] = red(cam.pixels[index]);
    col[1] = green(cam.pixels[index]);
    col[2] = blue(cam.pixels[index]);
  }
  return col;
}
 
public void rawData(float[] raw) {
  rawArray = raw; // stash data in array
}