nerual-Body

It's clouded out there. Why go outside when you can vicariously experience the great outdoors as a cloud? Turn your head and blow yourself back and forth.

Video:

GIFs:

Image:

Process:

I really like clouds. One of the commonalities among some examples of face tracking I've seen is giving non-human objects human qualities, so I thought I'd try it with clouds. I was interested in a fun and cute interaction that lasted a few minutes.

These are some initial sketches and ideas for doing more than just blowing yourself around as a cloud. Unfortunately I didn't get to much beyond that.

I used this p5.js with Glitch and clmTracker template provided. I used Matter.js for the physics, although in rettrospect, the p5.js built in physics would have sufficed.

The CLM tracker gets an array of coordinates from the video input. Here is a screen shot I used to know which points went with which features, so I could isolate the eyes and mouth and calculate ratios to determine if the mouth was open or which way the face was facing.

It was very difficult to get the motion tracker to stabilize and reliably detect movements, so I didn't get around to implementing networking or my other ideas.

As I was transplanting the face points, I discovered some quirky movements that resulted, such as the interesting face created when the eye position, but not rotation, is tracked. It gave the cloud(I call him Claude) a wonky face that I liked. I originally left the mouth as a line/circle(like in the sketches), but I wanted to capture more of the user's orientation and movement, and let it dictate rather than just guide the movement. In general, I liked this aesthetic that arose from selectively misinterpreting and/or ignoring certain features.

Ultimately however, I feel like this project was a disappointment. Part of it was me not taking the time to set up the right structures to handle a bunch of bodies interacting with physics. But another part might have been deciding to go with an idea that relied mostly on a smooth final result, rather than having other aspects to lean on. I think I made some decent efforts with adding in the other clouds, but their movement(move way too fast when video input isn't showing), and in general the color scheme, are pretty lacking.

Code:

/* face-ctrl.js */
var ctracker;
//var mouth_pos;
var mouth_x = 300;
var mouth_y = 300;
var facingLeft, facingRight;
var blowRatio, turnRatio;
var m_width, m_height;
 
var ox,oy;
var prev;
var leye, reye;
var pos1 = [];
var pos2 = [];
var posMid = [];
 
function setup_clm(){
  //setup camera capture
  var videoInput = createCapture(VIDEO);
  videoInput.size(w/4,h/4); 
  //uncomment to view video
  //videoInput.position(w/4*3, h/4*3);
  //videoInput.hide();
 
  ctracker = new clm.tracker();
  ctracker.init(pModel);
  ctracker.start(videoInput.elt);
  mouth_x = 300;
  mouth_y = 300;
}
 
 
 
function updateVars(mx, my, scale){
  var positions = ctracker.getCurrentPosition();
 
  for (var i=positions.length-1; i<positions.length; i++) { ox = positions[60][0]*scale; oy = positions[60][1]*scale; m_width = (positions[44][0] - positions[50][0]); m_height = (positions[47][1] - positions[53][1]); turnRatio = ((positions[44][0]-positions[1][0])/(positions[13][0]-positions[50][0])); facingLeft = turnRatio > 1.5;
      facingRight = turnRatio < 1;
  }
}
 
function isBlowing(){
  return blowRatio < 2.5;
}
 
function drawBlowHole(mx, my, scale){
    var positions = ctracker.getCurrentPosition();
  blowRatio = m_width/m_height;
 
  push();
  noFill();
  strokeWeight(5);
  var mw = m_width*scale;
  var mh = m_height*scale;
 
  if (isBlowing()){
    //wind
    stroke(0);
    var bx1 = mx + (facingLeft ? -1 : 1)*mw*2;
    var bx2 = mx + (facingLeft ? -1 : 1)*mw*3;
    var by = my;
    var sep = 20;
    push();
    strokeWeight(3);
    line(bx1, by, bx2, by);
    line(bx1, by+sep, bx2, by+sep);
    line(bx1, by-sep, bx2, by-sep);
    pop();
  }
 
  //face
  for (var i=0; i<positions.length; i++) {
    var x = positions[i][0]*scale;
    var y = positions[i][1]*scale;
    push();
    stroke(0);
    strokeWeight(scale*1.2);
    strokeJoin(ROUND);
    strokeCap(ROUND);
    //EYES
    if (i == 27 || i==32){
      stroke(0);
      noFill();
      pos1[0]= mx + x-ox;
      pos1[1] = my + y-oy +50;
      ellipse(pos1[0],pos1[1],40,20);
      fill(0);
      noStroke();
      ellipse(pos1[0],pos1[1],20,20);
    }
 
    //REAL MOUTH
    if(i<44 || i>61) continue;
    if (i==58 && !isBlowing()){
      i=62;
      continue;
    }
    if (i == 55) pos2 = [positions[44][0]*scale,positions[44][1]*scale];
    else if (i == 61) pos2 = [positions[56][0]*scale,positions[56][1]*scale];
    else pos2 = [positions[i+1][0]*scale,positions[i+1][1]*scale,x,y];
 
    pos1[0]= mx + x-ox;
    pos1[1] = my + y-oy;
    pos2[0] = mx + pos2[0]-ox;
    pos2[1] = my +pos2[1]-oy;
    posMid[0] = (pos1[0] + pos2[0])/2;
    posMid[1] = (pos1[1] + pos2[1])/2;
 
    line(pos1[0],pos1[1],posMid[0],posMid[1]);
    line(pos2[0],pos2[1],posMid[0],posMid[1]);
  }
  pop();
}
 
/* sketch.js */
 
 
var socket = io();
var clients = {};
var data = {};
var w = 800;
var h = 600;
var size;
var videoInput;
var ctracker;
var claude;
var clouds = [{x:100,y:100,scalar:100, w:3, h:4},
              {x:200, y:h-100, scalar:-150, w:5, h:7},
              {x:w-300, y:h, scalar: 70, w:8, h:5},
              {x:w-100, y:200, scalar: -50, w:6, h:6}
             ]
var t = 0;
var Engine = Matter.Engine,
    Render = Matter.Render,
    World = Matter.World,
    Bodies = Matter.Bodies,
    Body = Matter.Body,
    Constraint = Matter.Constraint;
 
function setup_matterjs(){
  engine = Engine.create();
  world = engine.world;
  engine.world.gravity.y = 0;
  engine.timing.timeScale = 0.2;
  //makeBoundaries(width,height);
  Engine.run(engine);
}
 
function setup() {
  // setup canvas
  var cnv = createCanvas(w,h);
  cnv.position(0, 0);
  setup_clm(); //face-ctrl
  setup_matterjs();
  size = 50;
  claude = new Cloud(mouth_x, mouth_y, size);
  World.add(world, claude.body);
  started=0;
}
 
function drawCloudShape(x,y,h,w){  
  push();
  fill(250, 250, 250);
  noStroke();
  arc(x, y, 25 * h, 20 * h, PI + TWO_PI, TWO_PI);
  arc(x + 10*w, y, 25 * h, 45 * h, PI + TWO_PI, TWO_PI);
  arc(x + 25*w, y, 25 * h, 35 * h, PI + TWO_PI, TWO_PI);
  arc(x + 40*w, y, 30 * h, 20 * h, PI + TWO_PI, TWO_PI);
  pop();
}
 
function drawCloud(x,y,w,h){
  updateVars(x,y,w/2);
  drawCloudShape(x-w*10,y+h*7,w,h);
  drawBlowHole(x,y,w/2);
}
 
function drawSun(){
  push();
  noStroke();
  fill('#FFEC5E');
  var sun =  {x: map(hour(), 0, 60, 150, w-150), y: 150, r:200};
  ellipse(sun.x, sun.y, sun.r);
  pop();
}
 
function draw() {
  background('#A6D7FF');  
  drawSun();
  sass();
 
  push();
  //uncomment to mirror
  //translate(w,0);
  //scale(-1.0,1.0);
  strokeWeight(1);
  claude.update;
  drawCloud(claude.pos.x, claude.pos.y, 7, 5);
  pop();
 
  var wind = 0.1 * (facingLeft ? -1 : 1);
  var force = {x:wind,y:0};
  if (blowRatio < 2) Body.applyForce(claude.body, claude.pos, force);
  for(var i =0; i< clouds.length; i++) otherClouds(clouds[i]); } function sass(){ push(); if (claude.pos.x > w+200 || claude.pos.x < -200){
      textFont('Georgia');
      textSize(30);
      textAlign(CENTER);
      strokeWeight(1);
      fill(0);
      text("the real world doesn't have wrap-arounds, chump\n go outside",w/2,h/2);
    }
  pop();
}
 
function makeBoundaries(w,h){
  console.log("width="+str(w)+" height="+str(h));
  var strength = 20;
  var options = {isStatic:true, restitution: 0.2};
  var bottom = Bodies.rectangle(w/2, h, w+50, strength, options);
  var top = Bodies.rectangle(w/2, 0, w+50, strength, options);
  var left = Bodies.rectangle(0, h/2, strength, h+50, options);
  var right = Bodies.rectangle(w, h/2, strength, h+50, options);
  World.add(world,top);
  World.add(world,bottom);
  World.add(world,left);
  World.add(world,right);
}
 
function otherClouds(cloud){
  var scalar = cloud.scalar;
  var x1 = cloud.x + (scalar * cos(t));
  drawCloudShape(x1, cloud.y, cloud.w, cloud.h);
  //fill(0); noStroke();
  stroke(0); noFill();
  var dir = cloud.scalar < 0 ? 1 : -1;
  var y = cloud.y-cloud.h*4;
  ellipse(x1 + cloud.w*15, y,20,10);
  ellipse(x1 + cloud.w*3, y, 20, 10);
  //ellipse(x1, cloud.y, 70, 70); //mouth?
  t+=0.005;
}