Happy No Grumpy from Chloe Chia on Vimeo.

As a favor for a friend obsessed with Grumpy Cat, I decided I would bring the meme to life by having him appear and judge the participant whenever a ‘smile’ was detected. After all, I’ve always been curious about the cutesy ‘smile detection technologies’ that some consumer cameras have, where the picture is only taken when all the faces detected in it are smiling. In my project, a ‘smile’ was determined by a rather crude set of ratios between the heights and width of the mouth in proportion to other properties on the face. Other data points used from the OSC included positioning and size, which helped determine the nature of the ‘neutral’ character, as well as the positioning of Grumpy Cat’s eyes, which in person, makes it seem as though he is looking at you regardless of how you are positioned towards the computer.

If it weren’t for time constraints I think I would’ve loved to pursue this further, smoothen the performance of the smile detection, and perhaps making use of OSC’s 3D capabilities to make for more a more natural neutral character, as well as the shifting of Grumpy Cat so that his judgement may rain ever more accurately.

(The video was laggy because my computer was having trouble handling Processing, FaceOSC, and video capture at the same time. Welp.)

I bit off a bit more than I can chew with trying to do things in 3D. I thought it would be cool to make a 3D game controlled by head movements that makes use of 3-D information provided by FaceOSC. The idea is to have a critter run on a grid and fire beams from its mouth to destroy obstacles. Properties used: face rotation, mouth height. The design of Kimpi (the critter) is supposed to include more complicated patterns, but I haven’t quite figured out how to draw them on correctly yet (too much math and pixel-manipulating). After making the general movements work, I realized that my visions of this game doesn’t really fit using FaceOSC very well – FaceOSC loses track of face easily, especially when the face is turned too much, so it is not fit for a slightly fast-paced game I wanted.

Task queue if/when time/interest allows/persists. As I was typing this, I realized this is way too ambitious even if I duplicated myself for the sole completion of this task:
– make Kimpi bounce up/down instead of glide (shouldn’t be hard)
– draw designs on Kimpi
– make keyboard controls
– include board tilt – Kimpi slides rapidly to one side
– include obstacles that can be destroyed when beam lands on them
– include enemies that actively attack Kimpi (flocking?) that can be destroyed by beam

Code for Kimpi Demo, Kimpi outward of screen and mirrors head movements. Made for testing Kimpi object class:

import oscP5.*;
OscP5 oscP5;

int     found;
PVector poseOrientation = new PVector();
float mouthHeight;

Kimpi kimpi = new Kimpi(50);

void setup() {
  size(400, 400, OPENGL, P3D);
  background(200);
  oscP5 = new OscP5(this, 8338);
  oscP5.plug(this, "found", "/found");
  oscP5.plug(this, "poseOrientation", "/pose/orientation");
  oscP5.plug(this, "mouthHeightReceived", "/gesture/mouth/height");

}
  
void draw() {
  background(200);
  translate(width/2, height/2, 0);
  spotLight(255, 255, 255, width/2, height/4, height*2, 0, 0, -1, PI/4, 2);

  if (found>0) {
    rotateY (poseOrientation.y); 
    rotateX (0-poseOrientation.x); 
    rotateZ (0-poseOrientation.z);
    println(mouthHeight);
  }


  kimpi.update(mouthHeight);
  kimpi.drawKimpi();

}


//----------------------------------
public void found (int i) { 
  found = i; 
}
public void poseOrientation(float x, float y, float z) {
  poseOrientation.set(x, y, z);
}
public void mouthHeightReceived(float h) {
  println("mouth height: " + h);
  mouthHeight = h;
}

class Kimpi{
  float radius;
  float headpieceW;
  float headpieceL;
  float mouthR = 0;
  float mouthRmax = 6;
  float eyeH;
  PFont f = createFont("Courier New Bold",16,true);
  
  Kimpi(float r){
    radius = r;
    headpieceW = radius/2;
    headpieceL = 5*headpieceW;
    eyeH = radius/12;
  }
  
  void update(float r){
    if (r<0) {
      mouthR = 0;
    }
    else if (r>mouthRmax) {
      mouthR = mouthRmax;
    }
    else {
      mouthR = r;
    }
  }

  void drawKimpi(){
    noStroke();
    fill(255,255,255);
    sphere(radius);
    
    float hx = 0;
    float hy = -radius;
    float hz = 0;
    
    pushMatrix();
    noFill();
    stroke(0,0,255,200);
    int nLines = 30;
    for (int i=0; i0) {
      rotateX(-PI/6);
      stroke(100);
      strokeWeight(3);
      noFill();
      beginShape();
      float theta = map((mouthR-1)/mouthRmax,0,mouthRmax,0,PI);
      for (int i=0; i< =20; i++){
        float phi = map(i,0,20,0,TWO_PI);
        float x = radius * sin(theta) * cos(phi); 
        float y = radius * sin(theta) * sin(phi); 
        float z = radius * cos(theta/2);
        vertex(x, y, z);
      }
      endShape();
      
      if ((mouthR-2>0)){
        float temp;
        if (mouthR-2<0) {
          temp = 0;
        } else {
          temp = mouthR-2;
        }
        stroke(200,220,255,200);
        float theta1 = map(temp/mouthRmax,0,mouthRmax,0,PI);        
        for (int i=0; i< =20; i++){
          float phi1 = map(i,0,20,0,TWO_PI);
          float x1 = radius * sin(theta1) * cos(phi1); 
          float y1 = radius * sin(theta1) * sin(phi1); 
          float z1 = radius * cos(theta1/2);
        
          line(x1,y1,z1,x1,y1-height/6,z1+height);
        }
      }
    }
    popMatrix();
  }
}

Code for Kimpi Beam, Kimpi faces into screen and travels on a grid:

import oscP5.*;
OscP5 oscP5;

int found;
PVector poseOrientation = new PVector();
float mouthHeight;

Kimpi kimpi = new Kimpi(20);

float unitSize = 30;
int nFrames0 = 50;
int nFrames = nFrames0;
float VX;
float VZ;
float maxV = 2;

Grid floor;

void setup() {
  size(800, 400, P3D);
  background(255,255,255);
  //  colorMode(HSB, 100);  
  floor = new Grid(width/2,height/5);
  
  oscP5 = new OscP5(this, 8338);
  oscP5.plug(this, "found", "/found");
  oscP5.plug(this, "poseOrientation", "/pose/orientation");
  oscP5.plug(this, "mouthHeightReceived", "/gesture/mouth/height");
  
  VX = sin(0)*maxV;
  VZ = cos(0)*maxV;
}

void draw() {
  background(255,255,255);
  
  float percentComplete = (float)(frameCount%nFrames)/ (float)nFrames;
  float runPercent = percentComplete;

  pushMatrix();
  translate(width/2,height/2,0);
  rotateX(PI/2.2);
  if (found>0) {
    VX = -sin(poseOrientation.y)*maxV;
    VZ = cos(poseOrientation.y)*maxV;
  }
  else {
    VX = sin(0)*maxV;
    VZ = cos(0)*maxV;
  }
  floor.drawGrid(percentComplete);
  popMatrix();
  
  
  pushMatrix();
  noStroke();
  fill(255,255,255);
  translate(width/2, 8.3*height/16, height/2);
  spotLight(255, 255, 255, width/2, height/4, height*2, 0, 0, -1, PI/4, 2);
  scale(1,1,-1);
  rotateX(PI/6);
  if (found>0) rotateY(poseOrientation.y);
  kimpi.update(mouthHeight);
  kimpi.drawKimpi();
  popMatrix();

}


class Grid {
  float left;
  float top;
  float currTX = 0;
  float currTZ = VZ;
  int dimension = int(2*width/unitSize);

  Grid(float cx, float cy) {
    left = cx-3*width/2;
    top = cy-height/1.5;
  }

  void drawGrid(float percent) {
    currTX+=VX;
    currTZ+=VZ;
    if (currTX>unitSize) currTX-=unitSize;
    if (currTZ>unitSize) currTZ-=unitSize;
    pushMatrix();
    translate(currTX,currTZ,0);
    stroke(0,240,255);
    fill(0);
    for (int i=0; imouthRmax) {
      mouthR = mouthRmax;
    }
    else {
      mouthR = r;
    }
  }

  void drawKimpi(){
    noStroke();
    fill(255,255,255);
    sphere(radius);
    
    float hx = 0;
    float hy = -radius;
    float hz = 0;
    
    pushMatrix();
    noFill();
    stroke(0,0,255,200);
    int nLines = 30;
    for (int i=0; i0) {
      rotateX(-PI/6);
      stroke(100);
      strokeWeight(3);
      noFill();
      beginShape();
      float theta = map((mouthR-1)/mouthRmax,0,mouthRmax,0,PI);
      for (int i=0; i< =20; i++){
        float phi = map(i,0,20,0,TWO_PI);
        float x = radius * sin(theta) * cos(phi); 
        float y = radius * sin(theta) * sin(phi); 
        float z = radius * cos(theta/2);
        vertex(x, y, z);
      }
      endShape();
      
      if ((mouthR-2>0)){
        float temp;
        if (mouthR-2<0) {
          temp = 0;
        } else {
          temp = mouthR-2;
        }
        stroke(200,220,255,200);
        float theta1 = map(temp/mouthRmax,0,mouthRmax,0,PI);        
        for (int i=0; i< =20; i++){
          float phi1 = map(i,0,20,0,TWO_PI);
          float x1 = radius * sin(theta1) * cos(phi1); 
          float y1 = radius * sin(theta1) * sin(phi1); 
          float z1 = radius * cos(theta1/2);
        
          line(x1,y1,z1,x1,y1-height/6,z1+height);
        }
      }
    }
    popMatrix();
  }
}

Defense Mechanisms

My initial idea was to create an onscreen character which takes a neutral/unfriendly expression and exaggerates it, literally reflecting the person’s “prickliness” or unapproachability. When a person smiles, then the character becomes rounded and happier.

In the end, while I worked on the sketch, I modified the concept. It became more of a creature and less of a puppet. The character becomes rounder and more visible when you smile, and pricklier and less visible for the longer you frown.  It also moves away toward the corner of the window for the longer you frown. Altogether, the character reacts to your expressions (or reflects your emotions, depending how you interpret it) by becoming more or less defensive (reflected in visibility, proximity, and prickliness.)

If I had more time to spend on this sketch, I would have experimented with moving it back toward my original concept—adding back facial features and making this more like a puppet.

Code

Github Repo

pricklyFace (main)

import oscP5.*;
OscP5 oscP5;

// our FaceOSC tracked face dat
Face face = new Face();
float faceScale = 1; // default - no resizing of face
ArrayList faceOutline = new ArrayList();
int numPoints = 100;
float initialPrickliness = 0.2;
float prickliness = initialPrickliness;
float maxPrickliness = 0.7;
float minPrickliness = 0;

float closeness = 0.3;
float maxCloseness = 0.7;
float minCloseness = 0.15;

void setup() {
  // default size is 640 by 480
  int defaultWidth = 640;
  int defaultHeight = 480;

  faceScale = 1; // shrink by half

  int realWidth = (int)(defaultWidth * faceScale);
  int realHeight = (int)(defaultHeight * faceScale);
  size(realWidth, realHeight, OPENGL);

  frameRate(10);

  oscP5 = new OscP5(this, 8338);
}

void draw() {  
  background(250);
  noStroke();

  updatePrickliness();

  if (face.found > 0) {
    
    // draw such that the center of the face is at 0,0
    translate(face.posePosition.x*faceScale*closeness, face.posePosition.y*faceScale*closeness);

    // scale things down to the size of the tracked face
    // then shrink again by half for convenience
    
    closeness = map(prickliness, minPrickliness, maxPrickliness, maxCloseness, minCloseness);
    scale(face.poseScale*closeness);

    // rotate the drawing based on the orientation of the face
    rotateY (0 - face.poseOrientation.y); 
    rotateX (0 - face.poseOrientation.x); 
    // rotateZ (    face.poseOrientation.z); 

    float fill = map(prickliness, minPrickliness, maxPrickliness, 100, 200);
    fill((int)fill);
    
    // drawEyes();
    // drawMouth();
    // print(face.toString());

    faceOutline = new ArrayList();
    getFaceOutlinePoints();
    drawOutline();
    
    /*if (face.isBlinking()) {
      println("BLINKED");
    }

    face.lastEyeHeight = face.eyeLeft;
    face.lastEyebrowHeight = face.eyeRight;
    */
  }
}

// OSC CALLBACK FUNCTIONS

void oscEvent(OscMessage m) {
  face.parseOSC(m);
}

void drawOutline() {
  float x = 0;
  float y = 0;

  if (faceOutline.size() != (numPoints + 1)) {
    getFaceOutlinePoints();
    return;
  }
  else {
    beginShape();
    for (int i=0; i < = numPoints; i++) {
      x = faceOutline.get(i).x;
      y = faceOutline.get(i).y;
      vertex(x, y);
    }  
    endShape();
  }

}

void updatePrickliness() {
  float antiPrickliness = 0;
  int transitionTime = 30000;

  if (!face.isSmiling()) {
    prickliness = constrain(face.timeSinceSmile, 0, transitionTime);
    prickliness = map(prickliness, 0, transitionTime, minPrickliness, maxPrickliness);
  }
  
  antiPrickliness = constrain(face.smilingTime, 0, transitionTime);
  antiPrickliness = -1 * map(antiPrickliness, 0, transitionTime, minPrickliness, maxPrickliness);
  
  prickliness = prickliness + antiPrickliness;
  constrain(prickliness, minPrickliness, maxPrickliness);
  if (prickliness < 0) {
    prickliness = 0;
  }
}

void getFaceOutlinePoints() {
  int xCenter = 0;
  int yCenter = 0;
  
  for (int i=0; i <= numPoints; i++) {
    float radius = 30;
  
    // iterate and draw points around circle
    float theta = 0;
    float x;
    float y; 
    float oldRadius = -1;
  
    theta = map(i, 0, numPoints, 0, 2*PI);
  
    if (i%2 == 0) {
      oldRadius = radius;
      radius = radius * random(1+prickliness, 1+(prickliness*2));
    }
  
    x = radius*cos(theta) + xCenter;
    y = radius*sin(theta) + yCenter;
  
    if (i == numPoints +1) {
      PVector firstPoint = faceOutline.get(0);
      PVector circlePoint = new PVector(firstPoint.x, firstPoint.y);
      faceOutline.add(circlePoint);
    } 
    else {
      PVector circlePoint = new PVector(x, y);
      faceOutline.add(circlePoint);
    }
  
    if (oldRadius > 0) {
      radius = oldRadius;
      oldRadius = -1;
    }
  }
}

void drawEyes() {
  int distanceFromCenterOfFace = 14;
  int heightOnFace = -4;
  int eyeWidth = 6;
  int eyeHeight = 4;
  ellipse(-1*distanceFromCenterOfFace, face.eyeLeft * heightOnFace, eyeWidth, eyeHeight);
  ellipse(distanceFromCenterOfFace, face.eyeRight * heightOnFace, eyeWidth, eyeHeight);
}

void drawMouth() {
  float mouthWidth = 30;
  int heightOnFace = 14;
  int mouthHeightFactor = 3;

  float mLeftCornerX = 0;
  float mLeftCornerY = heightOnFace;

  float pointX = mLeftCornerX + ((mouthWidth/2));

  float mouthHeight = face.mouthHeight * mouthHeightFactor;
  ellipse(mLeftCornerX, mLeftCornerY, mouthWidth, mouthHeight);
}

Face class

import oscP5.*;

// a single tracked face from FaceOSC
class Face {

  // num faces found
  int found;

  // pose
  float poseScale;
  PVector posePosition = new PVector();
  PVector poseOrientation = new PVector();

  // gesture
  float mouthHeight, mouthWidth;
  float eyeLeft, eyeRight;
  float eyebrowLeft, eyebrowRight;
  float jaw;
  float nostrils;

  // past
  float lastEyeHeight;
  float lastEyebrowHeight;
  
  boolean wasSmiling = false;
  float startedSmilingTime = 0;
  float smilingTime = 0;
  
  float stoppedSmilingTime = 0;
  float timeSinceSmile = 10000;

  Face() {
  }

  boolean isSmiling() {

    if (mouthIsSmiling()) {
      if (wasSmiling == false) {
        wasSmiling = true;
        startedSmilingTime = millis();
        timeSinceSmile = 0;
      }
      else {
        smilingTime = millis() - startedSmilingTime;
        println("smilingTime: ");
        print(smilingTime);
        println("");
      }
      return true;
    }
    else {
      if (wasSmiling == false) {
        timeSinceSmile = millis() - stoppedSmilingTime;
      }
      else {
        wasSmiling = false;
        stoppedSmilingTime = millis();
        smilingTime = 0;
      }
      return false;
    }
  }
  
  boolean mouthIsSmiling() {
    float minSmileWidth = 15;
    float minSmileHeight = 2;
    return ((mouthWidth > minSmileWidth) && (mouthHeight > minSmileHeight));
  }
  
  boolean isBlinking() {
    float eyeHeight = (face.eyeLeft + face.eyeRight) / 2;
    float eyebrowHeight = (face.eyebrowLeft + face.eyebrowRight) / 2;

    if ((eyeHeight < lastEyeHeight) &&
      (eyebrowHeight > lastEyebrowHeight)) {
      return true;
    }
    return false;
  }

  boolean isSpeaking() {
    int speakingMouthHeightThreshold = 2;
    if (face.mouthHeight > speakingMouthHeightThreshold) {
      return true;
    } 
    else {
      return false;
    }
  }

  // parse an OSC message from FaceOSC
  // returns true if a message was handled
  boolean parseOSC(OscMessage m) {

    if (m.checkAddrPattern("/found")) {
      found = m.get(0).intValue();
      return true;
    }      

    // pose
    else if (m.checkAddrPattern("/pose/scale")) {
      poseScale = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/pose/position")) {
      posePosition.x = m.get(0).floatValue();
      posePosition.y = m.get(1).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/pose/orientation")) {
      poseOrientation.x = m.get(0).floatValue();
      poseOrientation.y = m.get(1).floatValue();
      poseOrientation.z = m.get(2).floatValue();
      return true;
    }

    // gesture
    else if (m.checkAddrPattern("/gesture/mouth/width")) {
      mouthWidth = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/mouth/height")) {
      mouthHeight = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/eye/left")) {
      eyeLeft = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/eye/right")) {
      eyeRight = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/eyebrow/left")) {
      eyebrowLeft = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/eyebrow/right")) {
      eyebrowRight = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/jaw")) {
      jaw = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/nostrils")) {
      nostrils = m.get(0).floatValue();
      return true;
    }

    return false;
  }

  // get the current face values as a string (includes end lines)
  String toString() {
    return "found: " + found + "\n"
      + "pose" + "\n"
      + " scale: " + poseScale + "\n"
      + " position: " + posePosition.toString() + "\n"
      + " orientation: " + poseOrientation.toString() + "\n"
      + "gesture" + "\n"
      + " mouth: " + mouthWidth + " " + mouthHeight + "\n"
      + " eye: " + eyeLeft + " " + eyeRight + "\n"
      + " eyebrow: " + eyebrowLeft + " " + eyebrowRight + "\n"
      + " jaw: " + jaw + "\n"
      + " nostrils: " + nostrils + "\n";
  }
};

Being Shushed

My second idea was to create a character and, to some degree, an environment/game mechanic. When you open your mouth, a small speech bubble appears and begins to grow. However, as soon as you open your mouth, the word “shhhhh” begins to appear all around, and the words cluster around the speech bubble, as though they are squishing it. If you close your mouth, the speech bubble disappears and the face onscreen looks somewhat unhappy. But if you keep your mouth open long enough, the bubble grows and pushes the shhh’es out of the frame. If you successfully do this, you see the word applause appear all around.

I attempted to implement this idea and part of the way. I created (as shown in the video above) a speech bubble which grows based on how long you have been “speaking” (crudely measured by the length of time which you have had your mouth open). However, I had trouble figuring out how to position the face and speech bubble on screen such that they wouldn’t overlap awkwardly. I also realized that implementing some sort of particle system (most likely) of “shhh”es to put pressure on the speech bubble was going to make realizing this fully take a ton more time.

If I had more time to spend on this, I would probably stop drawing the face temporarily and work on the speech bubble’s interaction with a particle system of “shhh”es, then come back to the issue of the speaker’s face.

Code

Github Repo

shhhFace

//
// a template for receiving face tracking osc messages from
// Kyle McDonald's FaceOSC https://github.com/kylemcdonald/ofxFaceTracker
//
// this example includes a class to abstract the Face data
//
// 2012 Dan Wilcox danomatika.com
// for the IACD Spring 2012 class at the CMU School of Art
//
// adapted from from Greg Borenstein's 2011 example
// http://www.gregborenstein.com/
// https://gist.github.com/1603230

import oscP5.*;
OscP5 oscP5;

// our FaceOSC tracked face dat
Face face = new Face();
SpeechBubble speechBubble = new SpeechBubble();
float faceScale = 1;

// for additions


void setup() {
  // default size is 640 by 480
  int defaultWidth = 640;
  int defaultHeight = 480;
  
  int realWidth = (int)(defaultWidth * faceScale);
  int realHeight = (int)(defaultHeight * faceScale);
  size(realWidth, realHeight, OPENGL);
  
  frameRate(30);

  oscP5 = new OscP5(this, 8338);
}

void draw() {  
  background(255);
  stroke(0);

  if (face.found > 0) {
    
    // draw such that the center of the face is at 0,0
    translate(face.posePosition.x*faceScale, face.posePosition.y*faceScale);
    
    // scale things down to the size of the tracked face
    // then shrink again by half for convenience
    scale(face.poseScale*0.5);
    
    // rotate the drawing based on the orientation of the face
    rotateY (0 - face.poseOrientation.y); 
    rotateX (0 - face.poseOrientation.x); 
    rotateZ (    face.poseOrientation.z); 
    
    noFill();
    drawEyes();
    drawMouth();
    
    face.isSpeaking();
    int sbX = 7;
    int sbY = -15;
    speechBubble.draw(sbX, sbY);
      
    //}
    
    //drawNose();
    //drawEyebrows();
    print(face.toString());
    
    if (face.isSmiling()) {
      println("SMILING");
    }
    if (face.isBlinking()) {
      println("BLINKED");
    }
    
    face.lastEyeHeight = face.eyeLeft;
    face.lastEyebrowHeight = face.eyeRight;
    println("lastEyeHeight " + face.lastEyeHeight);
    println("lastEyebrowHeight " + face.lastEyebrowHeight);
  }
}

// OSC CALLBACK FUNCTIONS

void oscEvent(OscMessage m) {
  face.parseOSC(m);
}

void drawEyes() {
  int distanceFromCenterOfFace = 20;
  int heightOnFace = -9;
  int eyeWidth = 6;
  int eyeHeight =5;
  ellipse(-1*distanceFromCenterOfFace, face.eyeLeft * heightOnFace, eyeWidth, eyeHeight);
  ellipse(distanceFromCenterOfFace, face.eyeRight * heightOnFace, eyeWidth, eyeHeight);
}
void drawEyebrows() {
  rectMode(CENTER);
  fill(0);
  int distanceFromCenterOfFace = 20;
  int heightOnFace = -5;
  int eyebrowWidth = 23;
  int eyebrowHeight = 2;
  rect(-1*distanceFromCenterOfFace, face.eyebrowLeft * heightOnFace, eyebrowWidth, eyebrowHeight);
  rect(distanceFromCenterOfFace, face.eyebrowRight * heightOnFace, eyebrowWidth, eyebrowHeight);
}
void drawMouth() {
  float mouthWidth = 30;
  int heightOnFace = 14;
  int mouthHeightFactor = 6;
  
  float mLeftCornerX = 0;
  float mLeftCornerY = heightOnFace;
 
  float pointX = mLeftCornerX + ((mouthWidth/2));
  
  float mouthHeight = face.mouthHeight * mouthHeightFactor;
  ellipse(mLeftCornerX, mLeftCornerY, mouthWidth, mouthHeight);
}

void drawNose() {
  int distanceFromCenterOfFace = 5;
  int heightOnFace = -1;
  int nostrilWidth = 4;
  int nostrilHeight = 3;
  ellipse(-1*distanceFromCenterOfFace, face.nostrils * heightOnFace, nostrilWidth, nostrilHeight);
  ellipse(distanceFromCenterOfFace, face.nostrils * heightOnFace, nostrilWidth, nostrilHeight);
}

SpeechBubble

class SpeechBubble {
  float xPos; 
  float yPos; 

  float sbHeight = 150*0.25;
  float sbWidth = 250*0.25;
 
  float initialRadius = (sbHeight/3);
  float radius = initialRadius;
 
  int numPoints = 30;
  // http://math.rice.edu/~pcmi/sphere/degrad.gif
  float extrusionTheta = (5*PI)/6;
  float epsilon = PI/25;
  
  void draw(float xPosition, float yPosition) {
    xPos = xPosition;
    yPos = yPosition;
    
    float timeRadiusFactor = face.totalTime/10000;
    
    radius = radius + timeRadiusFactor;
    
    if (radius < 10) {
      return;
    }
    
    float xCenter = xPos+sbWidth/2 + timeRadiusFactor;
    float yCenter = yPos+sbHeight/2 - (timeRadiusFactor/2);
    
    println("DRAWN");
    beginShape();
    
      // variables for calculating each point
      float x;
      float y;
      float theta;   
      
      // iterate and draw points around circle.
      for (int i = 0; i <= numPoints; i++) {
        
        theta = map(i, 0, numPoints-2, 0, 2*PI);
        // this minus-2 is a hack to make the circle close
        x = radius*cos(theta) + xCenter;
        y = radius*sin(theta) + yCenter;
        
        // check to see if we're at the point in the circle where 
        // we want to draw the part of the speech bubble that sticks out
        if (((theta - epsilon) < extrusionTheta) && 
            ((theta + epsilon) > extrusionTheta)){
             
              float extrusionRadius = PI/25;
              
              float startTheta = extrusionTheta - extrusionRadius;
              float endTheta = extrusionTheta + extrusionRadius;
              
              float startX = radius*cos(startTheta) + xCenter;
              float startY = radius*sin(startTheta) + yCenter;
  
              float endX = radius*cos(endTheta) + xCenter;
              float endY = radius*sin(endTheta) + yCenter;
            
              curveVertex(startX, startY);
              vertex(startX, startY);
              vertex(x - (radius/1.5), y+ (radius/3));
              vertex(endX, endY);
              curveVertex(endX, endY);
        }
        else {
          curveVertex(x, y);
        }
      }
    endShape();
  }
}

Face class

import oscP5.*;

// a single tracked face from FaceOSC
class Face {

  // num faces found
  int found;

  // pose
  float poseScale;
  PVector posePosition = new PVector();
  PVector poseOrientation = new PVector();

  // gesture
  float mouthHeight, mouthWidth;
  float eyeLeft, eyeRight;
  float eyebrowLeft, eyebrowRight;
  float jaw;
  float nostrils;

  // past
  float lastEyeHeight;
  float lastEyebrowHeight;
  boolean wasSpeaking = false;
  float startSpeakingTime = 0;
  float totalTime = 0;
  float stoppedSpeakingTime = 0;

  Face() {
  }

  boolean isSmiling() {
    float minSmileWidth = 15;
    float minSmileHeight = 2;

    if ((mouthWidth > minSmileWidth) &&
      (mouthHeight > minSmileHeight)) {
      return true;
    }
    return false;
  }

  boolean isBlinking() {
    float eyeHeight = (face.eyeLeft + face.eyeRight) / 2;
    float eyebrowHeight = (face.eyebrowLeft + face.eyebrowRight) / 2;

    if ((eyeHeight < lastEyeHeight) &&
      (eyebrowHeight > lastEyebrowHeight)) {
      return true;
    }
    return false;
  }

  boolean isSpeaking() {
    int speakingMouthHeightThreshold = 2;
    /* Debug: 
     println("MOUTHHEIGHT");
     println(face.mouthHeight);
     */
     println("totalTime: ");
     print(totalTime);
     println("");
     
    if (face.mouthHeight > speakingMouthHeightThreshold) {
      if (!wasSpeaking) {
        totalTime = 0;
        startSpeakingTime = millis();
        wasSpeaking = true;
      }
      else {
        totalTime = millis() - startSpeakingTime;
      }
      println("SPEAKING");
      return true;
    } 
    else {
      if (wasSpeaking) {
        println("NOT SPEAKING");
        stoppedSpeakingTime = millis();
        wasSpeaking = false;
        totalTime = 0;
      }
      else {
        totalTime = -1*(millis() - stoppedSpeakingTime);
      }
      return false;
    }
  }

  // parse an OSC message from FaceOSC
  // returns true if a message was handled
  boolean parseOSC(OscMessage m) {

    if (m.checkAddrPattern("/found")) {
      found = m.get(0).intValue();
      return true;
    }      

    // pose
    else if (m.checkAddrPattern("/pose/scale")) {
      poseScale = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/pose/position")) {
      posePosition.x = m.get(0).floatValue();
      posePosition.y = m.get(1).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/pose/orientation")) {
      poseOrientation.x = m.get(0).floatValue();
      poseOrientation.y = m.get(1).floatValue();
      poseOrientation.z = m.get(2).floatValue();
      return true;
    }

    // gesture
    else if (m.checkAddrPattern("/gesture/mouth/width")) {
      mouthWidth = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/mouth/height")) {
      mouthHeight = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/eye/left")) {
      eyeLeft = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/eye/right")) {
      eyeRight = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/eyebrow/left")) {
      eyebrowLeft = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/eyebrow/right")) {
      eyebrowRight = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/jaw")) {
      jaw = m.get(0).floatValue();
      return true;
    }
    else if (m.checkAddrPattern("/gesture/nostrils")) {
      nostrils = m.get(0).floatValue();
      return true;
    }

    return false;
  }

  // get the current face values as a string (includes end lines)
  String toString() {
    return "found: " + found + "\n"
      + "pose" + "\n"
      + " scale: " + poseScale + "\n"
      + " position: " + posePosition.toString() + "\n"
      + " orientation: " + poseOrientation.toString() + "\n"
      + "gesture" + "\n"
      + " mouth: " + mouthWidth + " " + mouthHeight + "\n"
      + " eye: " + eyeLeft + " " + eyeRight + "\n"
      + " eyebrow: " + eyebrowLeft + " " + eyebrowRight + "\n"
      + " jaw: " + jaw + "\n"
      + " nostrils: " + nostrils + "\n";
  }
};

Other Idea: Feeling Misinterpreted

One of my initial ideas was to create a face/character that would mirror your expressions but be…off. The face itself would be distorted, somewhat ugly, with some features upside-down or asymmetrical. As you looked at the face, it would mirror your expressions somewhat—if you smile, it would smile too, but crookedly, awkwardly.

The concept for this was to create a sort of mirror that evokes the feeling of being misinterpreted, not being able to say the right thing, or express it effectively.

I abandoned this idea because after initial experimentation, I decided that it would be too difficult to get to the point of accurately mirroring a face’s expressions so that I could deliberately distort parts of that mirroring.

This piece is about the impact that small gestures put together can have on the world. With just the subtle motion of blinking one’s eyes, the fire will react in an equally subtle way until, eventually, the fire grows too big and wild for the candle and burns out.
The program tracks the height of the user’s eyes, and when it detects that the eyes are closed, a disturbance is sent through the fire and the wildness, number of fire particles, and the height increases by a tiny margin.

The video shows the program less responsive than usual, as faceOSC had a poor time trying to differentiate between my eyes and eyebrows under the bad lighting. Also, my eyes are already tiny enough as is. I’ll try to make a better recording.

I made some generic character sketches but none of them felt compelling, so I tried to think more about the potential of the library. Then the character started out as a non-corporeal lifeform inspired by the ones in Star Trek (namely the one in The Day of the Dove, where George Takei swings around a sword half-naked). The Beta XII-A entity in the episode fed off humanoid emotions, but I wanted to make the creature friendly instead of malicious. As I kept moving around the form with my face, it almost looked like it was real, imitating my movement as if it were a curious being, and suddenly I could see it as something within the computer. The sounds made were inspired from various digitized creature noises I’ve come across before (can’t remember exactly where).

I wanted the work to be more delayed though so that it would actually look like the thing was trying to process your movements before proceeding to copy them.  It would also be nice to have more variety of sounds for the creature and give it more personality with more degree of control. The three OSC properties of the face I used was the position, scale, and mouth height. The main degree of control was on the mouth height, because then the change was drastic. I don’t particularly like what I made, mostly because I feel like there could’ve been more variations on the creature behavior, but I like the idea behind it.

note: the audio in the video was artificially put in because of complications with capturing the internal sounds using Quicktime. It definitely works in person, as demonstrated in class.

Sometimes, we forget to notice that the tiny common things around us can also be some of the most beautiful. Dust balls are often disliked due to their nature of sticking to things and cause of allergies. However, when shown in slow motion and against the light, their paths traveled through the air become quite alluring.

For my project, I wanted to simulate pollution as it flowed through the air. I got my inspiration both from my previous clock project and from the sootballs in the movie, Spirited Away. The way that the air emphasized the sootballs’ lightness captured my interest.

At first I mapped the size of the eyes to the arguments that faceOSC outputted. Every time something changed, the program would output a number for the change. I figured out the difference between my opened eyes and my closed eyes. Then, I changed the size of the soot balls based on my blinking. Using the same tactic as I did with my eyes, I mapped the force of the wind to the height of my mouth. I wanted to allow people to interact in a way that would be nature for them to do in real life with dust. Therefore, when one’s mouth would open as if to blow them away, the dust balls fly at a speed depending on how big the mouth is opened.

I think I managed the simulation of the lightness of the soot, but I wish I could have made them slowly flow upwards again after changing direction, instead of so abruptly. I also wish I could have added a windmill as a decoration on the side. This was a really fun project though,  and I learned a lot of about making classes and utilizing them through your draw function.

(video pending)

dust mites from Maryyann Landlord on Vimeo.

 

 

//
// a template for receiving face tracking osc messages from
// Kyle McDonald's FaceOSC https://github.com/kylemcdonald/ofxFaceTracker
//
// 2012 Dan Wilcox danomatika.com
// for the IACD Spring 2012 class at the CMU School of Art
//
// adapted from from Greg Borenstein's 2011 example
// http://www.gregborenstein.com/
// https://gist.github.com/1603230
//
// This is an adapted version of the template by Swetha Kannan

import oscP5.*;
OscP5 oscP5;
/* @pjs preload= "happy.jpg"; */
PImage bg;
float petalColor = 255.0;
float petalColor2 = 140.0;
float petalColor3 = 140.0;

// num faces found
int found;
float cx, cy;
float secondsRadius;

// pose
float poseScale;
PVector posePosition = new PVector();
PVector poseOrientation = new PVector();

// gesture
float mouthHeight;
float mouthWidth;
float eyeLeft;
float eyeRight;
float eyebrowLeft;
float eyebrowRight;
float jaw;
float nostrils;

void setup() {
  size(640, 480);
  frameRate(30);
  int radius = min(width, height) / 2;
  secondsRadius = radius * 0.2;

  oscP5 = new OscP5(this, 8338);
  oscP5.plug(this, "found", "/found");
  oscP5.plug(this, "poseScale", "/pose/scale");
  oscP5.plug(this, "posePosition", "/pose/position");
  oscP5.plug(this, "poseOrientation", "/pose/orientation");
  oscP5.plug(this, "mouthWidthReceived", "/gesture/mouth/width");
  oscP5.plug(this, "mouthHeightReceived", "/gesture/mouth/height");
  oscP5.plug(this, "eyeLeftReceived", "/gesture/eye/left");
  oscP5.plug(this, "eyeRightReceived", "/gesture/eye/right");
  oscP5.plug(this, "eyebrowLeftReceived", "/gesture/eyebrow/left");
  oscP5.plug(this, "eyebrowRightReceived", "/gesture/eyebrow/right");
  oscP5.plug(this, "jawReceived", "/gesture/jaw");
  oscP5.plug(this, "nostrilsReceived", "/gesture/nostrils");
}

void draw() {  
  background(200);
  stroke(0);
  bg = loadImage("flowers.jpg");
  image(bg, 0, 0);
  image(bg, 0, 0, width, height);

  if (found > 0) {

    cx = width / 2;
    cy = height - ((height -posePosition.y)/2);

    //stem will grow along with pose position
    stroke(19, 151, 37);
    strokeWeight(10);
    noFill();
    float bexierX = width/2+(width/2-posePosition.x); // will control the curve of the stem
    float bexier2X = width/2-(width/2-posePosition.x); 
    bezier(cx, cy, bexierX , cy, bexierX, height,  width/2, height);
    bezier(cx, posePosition.y,  bexier2X, posePosition.y, bexier2X, cy,  width/2, cy);

    //Leaf will grow along with stem
    pushMatrix();
    stroke(19, 151, 37);
    fill(19, 200, 37);
    translate(width/2-15, cy-40);
    print (int(posePosition.y/30));
    drawLeaf(int(20-(posePosition.y/30)));
    popMatrix();

    if (eyebrowLeft >= 8.5) {

      petalColor = random(0, 255);
      petalColor2 = random(0, 255);
      petalColor3 = random(0, 255);
    }

    //num of Petals will be dependant on how open mouth is
    strokeWeight(10);
    stroke(petalColor, petalColor2, petalColor3);
    strokeWeight((height - posePosition.y)/10);
    for (int petals = int(mouthHeight); petals>0; petals -=1) {
      int maxPetals =petals;
      float Spacing = TWO_PI / max;
      line(cx, int(posePosition.y), cx + cos(petals)*secondsRadius, int(posePosition.y) + sin(petals)*secondsRadius);
    }

    //middle yellow of flower will grow along with posePosition
    stroke(250, 200, 50);
    strokeWeight((height - posePosition.y)/5);
    point(width/2, posePosition.y);
  }
}

// OSC CALLBACK FUNCTIONS

//drawLeaf function borrowed from Cath at https://openprocessing.orgsketch/7743
//Thankyou, I claim no responsibility
void drawLeaf(int shift) { // draw a leaf as follows

  float pointShift = shift;
  beginShape(); // start to draw a shape
  vertex(20, 45); // begin at this point x, y
  bezierVertex(30, 30, 60 + pointShift, 40 + pointShift/2, 70 + pointShift, 50); 
  bezierVertex(60 + pointShift, 55, 30, 65, 20, 45); // draw the other half of the shape
  endShape();
}

public void found(int i) {
  println("found: " + i);
  found = i;
}

public void poseScale(float s) {
  println("scale: " + s);
  poseScale = s;
}

public void posePosition(float x, float y) {
  println("pose position\tX: " + x + " Y: " + y );
  posePosition.set(x, y, 0);
}

public void poseOrientation(float x, float y, float z) {
  println("pose orientation\tX: " + x + " Y: " + y + " Z: " + z);
  poseOrientation.set(x, y, z);
}

public void mouthWidthReceived(float w) {
  println("mouth Width: " + w);
  mouthWidth = w;
}

public void mouthHeightReceived(float h) {
  println("mouth height: " + h);
  mouthHeight = h;
}

public void eyeLeftReceived(float f) {
  println("eye left: " + f);
  eyeLeft = f;
}

public void eyeRightReceived(float f) {
  println("eye right: " + f);
  eyeRight = f;
}

public void eyebrowLeftReceived(float f) {
  println("eyebrow left: " + f);
  eyebrowLeft = f;
}

public void eyebrowRightReceived(float f) {
  println("eyebrow right: " + f);
  eyebrowRight = f;
}

public void jawReceived(float f) {
  println("jaw: " + f);
  jaw = f;
}

public void nostrilsReceived(float f) {
  println("nostrils: " + f);
  nostrils = f;
}

// all other OSC messages end up here
void oscEvent(OscMessage m) {

  /* print the address pattern and the typetag of the received OscMessage */
  println("#received an osc message");
  println("Complete message: "+m);
  println(" addrpattern: "+m.addrPattern());
  println(" typetag: "+m.typetag());
  println(" arguments: "+m.arguments()[0].toString());

  if (m.isPlugged() == false) {
    println("UNPLUGGED: " + m);
  }
}

In this project, I was attempting to make a portrait of someone as a flower. The flower reacts to you and expresses itself in similar ways to you. When, you are slumping the flower is small but as you sit up straight, the flower grows as well. The stem gets taller, the leaf bigger and the flower itself larger. Similarly, if your mouth is closed only one petal will grow on the flower but with your mouth open multiple petals will begin to bloom. The movement of your eyebrows will also affect the color of the plant; in essence you can keep raising your eyebrows until you find a color that suits you. The tilt and movements of your head can also affect the curving of the flower stem. The flower is only as expressive as you, the user are. This project, although simple, carries a lot of what my visual clock did in that it is meant as an appreciation of nature, a landscape, and a scene. I am very happy with how the visuals of the project has turned out. I was able to create a working portrait of a flower that would mimics the user. I will probably go back to this project in order to create more depth in the flower and more customization options so that the flower will be an unmistakable portrait of someone.

As Kyle McDonald so aptly puts it, the face is “one of the most salient objects in our day-to-day life.” It is arguably the most central means by which we communicate non-verbally with others, and is undoubtedly our most powerful tool for emotional expression. Given its primacy in these areas, I was interested in the history of the face in both art and science. After some research I stumbled upon the then science, now pseudoscience “Physiognomy”, which aims to judge one’s character and intellectual aptitude based on facial structure alone.

Physiognomy still going strong in North Korea, here applied to judge Kim Jong-Un’s leadership potential

While the field may not have contributed much to Western science, it did yield some beautiful illustrations.

One of Charles Le Brun’s (1619 – 1619) anamorphic physiognomies

 

Owl + Man in a fashionable orange jumpsuit

Tapping into the idea of a human-animal hybrid, I composited a half owl half man chimera in Photoshop. One can spy on this chimera through a keyhole. Although the face itself does not respond to the user, the scene does. By moving one’s head back and forth, ones changes the perspective from which the creature can be seen, thus the illusion of depth. This effect is called parallax, and the layers of images move at distinct speeds accordingly. I applied a fisheye filter to the assets that lie behind the keyhole, so as to give the impression of looking through a piece of curved glass.

Ideally, the face would respond to the user as well. However, for the purpose of this project, the parallax achieves some level of immersion on its own. I also think the voyeuristic quality of this piece reveals rich territory for future exploration/exploitation/exposition. 

I used FaceOSC’s poseOrientation property to govern the rotation of the head (a texture on a cylinder), the posePosition property to pan the images, and the poseScale property to control the zoom level of the scene.

polystag

Mention the specific OSC properties of the face that you used. How many dimensions of control does your design have?

I am inspired by low poly / delaunay raster type art and design –

I wanted to create a puppet in processing using this style and have each individual point in the image “ease” differently so that the whole face becomes kind of fluid like. Unfortunately I ran out of time to implement this.

I started with an image of a stag and used a program called DMesh to trace a mesh over it.

polystag from adambd on Vimeo.

The stag’s mouth height and width is controlled by the user’s mouth height and width, it’s eyes are mapped to the users. The stag is scaled and positioned according the user.
Finally it’s ears are controlled by the users eyebrows – Humans use their eyebrows to convey expression, stags use their ears. In this way my programme acts as an interspeciel translator.

//
// a template for receiving face tracking osc messages from
// Kyle McDonald's FaceOSC https://github.com/kylemcdonald/ofxFaceTracker
//
// 2012 Dan Wilcox danomatika.com
// for the IACD Spring 2012 class at the CMU School of Art
//
// adapted from from Greg Borenstein's 2011 example
// http://www.gregborenstein.com/
// https://gist.github.com/1603230
//
import oscP5.*;
OscP5 oscP5;

// num faces found
int found;
float y1; 

PImage img, cloud;


// pose
float poseScale;
PVector posePosition = new PVector();
PVector poseOrientation = new PVector();

// gesture
float mouthHeight;
float mouthWidth;
float eyeLeft;
float eyeRight;
float eyebrowLeft;
float eyebrowRight;
float jaw;
float nostrils;

void setup() {
  size(640, 480);
  //  frameRate(30);
  img = loadImage("2.png");
  cloud = loadImage("4.png");

  smooth();


  oscP5 = new OscP5(this, 8338);
  oscP5.plug(this, "found", "/found");
  oscP5.plug(this, "poseScale", "/pose/scale");
  oscP5.plug(this, "posePosition", "/pose/position");
  oscP5.plug(this, "poseOrientation", "/pose/orientation");
  oscP5.plug(this, "mouthWidthReceived", "/gesture/mouth/width");
  oscP5.plug(this, "mouthHeightReceived", "/gesture/mouth/height");
  oscP5.plug(this, "eyeLeftReceived", "/gesture/eye/left");
  oscP5.plug(this, "eyeRightReceived", "/gesture/eye/right");
  oscP5.plug(this, "eyebrowLeftReceived", "/gesture/eyebrow/left");
  oscP5.plug(this, "eyebrowRightReceived", "/gesture/eyebrow/right");
  oscP5.plug(this, "jawReceived", "/gesture/jaw");
  oscP5.plug(this, "nostrilsReceived", "/gesture/nostrils");
}

void draw() {  
  background(255);
  //image(img, 100, 0);
  //  image(cloud, 200, 100);



  noFill();
  stroke(100, 0, 0); 
  translate(width - posePosition.x, posePosition.y);

  //      line(posePosition.x, posePosition.y * -1, 0, 0);
  //      line(posePosition.x - width, posePosition.y * -1, 0, 0); 
  //      line(posePosition.x, height - posePosition.y, 0, 0);
  //      line(posePosition.x - width, height - posePosition.y, 0, 0);

  stroke(0); 
  scale(poseScale);
  rotate  (    poseOrientation.z * -1 +.08); 
  strokeWeight(.2); 

  //left horn
  beginShape();
  vertex(-14, -19);
  vertex(-15.5, -23);
  vertex(-35, -30);
  vertex(-42, -38);
  vertex(-45, -51.5);
  vertex(-43.5, -60);
  vertex(-43, -51.5);
  vertex(-36, -38);
  vertex(-32, -32);
  vertex(-10, -28);
  vertex(-7, -34);
  vertex(-2, -38);
  vertex(-2, -33);
  vertex(-7, -23);
  vertex(-7, -21.5);
  endShape(CLOSE);

  line(-15.5, -23, -10, -28);
  line(-35, -30, -32, -32);
  line(-10, -28, -30, -34);
  line(-30, -34, -32, -32);
  line(-30, -34, -36, -42);
  line(-36, -42, -37, -47);
  line(-36, -42, -37, -47);
  line(-37, -47, -37, -40);
  line(-42, -38, -42, -49.5);
  line(-10, -28, -10, -20.5);
  line(-10, -24, -7, -28);
  line(-7, -28, -6, -34.5);
  line(-7, -28, -6, -26);

  //Right horn
  beginShape();
  vertex(14, -19);
  vertex(15.5, -23);
  vertex(35, -30);
  vertex(42, -38);
  vertex(45, -51.5);
  vertex(43.5, -60);
  vertex(43, -51.5);
  vertex(36, -38);
  vertex(32, -32);
  vertex(10, -28);
  vertex(7, -34);
  vertex(4, -42);
  vertex(3, -33);
  vertex(7, -23);
  vertex(7, -21.5);
  endShape(CLOSE);

  line(15.5, -23, 10, -28);
  line(35, -30, 32, -32);
  line(10, -28, 30, -34);
  line(30, -34, 32, -32);
  line(30, -34, 36, -42);
  line(36, -42, 37, -47);
  line(36, -42, 37, -47);
  line(37, -47, 37, -40);
  line(42, -38, 42, -49.5);
  line(10, -28, 10, -20.5);
  line(7, -21.5, 6, -33);
  line(6, -33, 3.5, -35);


  //face outline 
  beginShape();
  vertex(-4.5, -22.5);
  vertex(-3, -21);
  vertex(3, -21);
  vertex(4.5, -22.5);
  vertex(14.5, -18.5);
  vertex(14, -17);
  vertex(18, -8);
  vertex(13, -2);
  vertex(14, 0);
  vertex(12, 3);
  vertex(12, 3);
  vertex(13, 10.5);
  vertex(11.5, 12);
  vertex(11, 18.5);
  vertex(6, 29);
  vertex(7-mouthWidth/2, 34+mouthHeight/2);
  vertex(4-mouthWidth/2, 34+mouthHeight/2);
  vertex(-6, 29);
  vertex(-11, 18.5);
  vertex(-11.5, 12);
  vertex(-13, 10.5);
  vertex(-12, 3);
  vertex(-12, 3);
  vertex(-14, 0);
  vertex(-13, -2);
  vertex(-18, -8);
  vertex(-14, -17);
  vertex(-14, -19);
  endShape(CLOSE);

  //    ellipse (-11.5, 12, 3,3); 

  line(-3.5, 10, 3.5, 10);

  //face fill left
  line(-14, -17, -8, -16);
  line( -8, -16, -3, -21);
  line(-8, -16, 0, -10);
  line(-8, -16, -13, (eyeLeft*-3)-1  );
  line(-13, (eyeLeft*-3)-1, -13.5, (eyeLeft*-3)+1.7  );
  line(-13.5, (eyeLeft*-3)+1.7, -17, -18  );
  line(-12, 3, 0-mouthWidth/2, 14.5-mouthHeight/2);
  line(0-mouthWidth/2, 14.5-mouthHeight/2, -8, 19);
  line (-8, 19, 0-mouthWidth/2, 22-mouthHeight/2);
  line (-8, 19, -11.5, 12);
  line (-11, 18.5, 2-mouthWidth/2, 25+mouthHeight/2);
  //  line ( 2-mouthWidth/2, 25+mouthHeight/2, 0, 24); 
  line (6, 29, 7.5, 30); 
  line(0-mouthWidth/2, 14.5-mouthHeight/2, -3.5, 10);
  line(-5, 6, -3.5, 10);
  line(-5, 6, -3, -.5);
  line(-3, -.5, 0, -2);
  line(-6.5, -15, -7, -2);
  line(-7, -2, -13.5, 2+(eyeLeft*-3) ); 
  line(-6.5, -15, -13.5, 2+(eyeLeft*-3)); 
  line( -12, eyeLeft-8, -7, -2);
  line( -12, eyeLeft-8, -9, 4);
  line( -9, 8, -9, 4);


  line(-3, -.5, -6.5, -15);
  line(-3, -.5, -7, -2);


  line(-5, 6, -9, 4);


  //face fill right
  line(14, -17, 8, -16);
  line( 8, -16, 3, -21);
  line(8, -16, 0, -10);
  line(8, -16, 13, (eyeRight*-3)-1  );
  line(13, (eyeRight*-3)-1, 13.5, (eyeRight*-3)+1.7  );
  line(13.5, (eyeRight*-3)+1.7, 17, -18  );
  line(+12, 3, 12-mouthWidth/2, 14.5-mouthHeight/2);
  line(12-mouthWidth/2, 14.5-mouthHeight/2, 8, 19);
  line (8, 19, 12-mouthWidth/2, 22-mouthHeight/2);
  line (8, 19, 11.5, 12);
  line (11, 18.5, 10-mouthWidth/2, 25+mouthHeight/2);
  //  line ( 10-mouthWidth/2, 25+mouthHeight/2, 0, 24); 
  line (-6, 29, -7.5, 30); 
  line(12-mouthWidth/2, 14.5-mouthHeight/2, 3.5, 10);
  line(5, 6, 3.5, 10);
  line(5, 6, 3, -.5);
  line(3, -.5, 0, -2);
  line(6.5, -15, 7, -2);
  line(7, -2, 13.5, 2+(eyeRight*-3) ); 
  line(6.5, -15, 13.5, 2+(eyeRight*-3)); 
  line( 12, eyeRight-8, 7, -2);
  line( 12, eyeRight-8, 9, 4);
  line( 9, 7.5, 9, 4);

  line(3, -.5, 6.5, -15);
  line(3, -.5, 7, -2);

  line(5, 6, 9, 4);


  line(11, 18.5, 8, 29);
  line(8, 29, 0, 50);
  line(0, 50, -8, 29);
  line(-8, 29, -11, 18.5);
  line(-11, 18.5, -10.2, 30);
  line(-10.2, 30, -7.2, 44);
  line(-7.2, 44, -3.5, 41);


  rectMode(CENTER);

  //eye brow left
  beginShape();
  vertex(-10 - (eyebrowLeft * 4), (eyebrowLeft * -5) +12 );
  vertex(-33, (eyebrowLeft * -5) +24.5);
  vertex(-22, (eyebrowLeft * -2) +8);
  vertex(-18, -8);
  vertex(-14, -17);
  vertex(-17, -18);
  vertex(-20, (eyebrowRight * -5) +14);
  endShape(CLOSE);

  line(-10 - (eyebrowLeft * 4), (eyebrowLeft * -5) +12, -22, (eyebrowLeft * -2) +8 );
  line(-10 - (eyebrowLeft * 4), (eyebrowLeft * -5) +12, -25, -15 );
  line(-25, -15, -4 - (eyebrowLeft * 4), (eyebrowLeft * -5) +13 );
  line(-25, -15, -18, -8);
  line(  -33, (eyebrowLeft * -5) +24.5, -23, -13);
  line(-23, -13, -23, (eyebrowRight * -5) +17);
  line(-23, (eyebrowRight * -5) +17, -20, (eyebrowRight * -5) +14);
  line(-4 - (eyebrowLeft * 4), (eyebrowLeft * -5) +13, -23, (eyebrowRight * -5) +17);
  line(-23, (eyebrowRight * -5) +17, -17, -18);
  line (-17, -18, -18, -8);



  //eye brow right
  beginShape();
  vertex(10+(eyebrowRight*4), (eyebrowRight * -5) +12 );
  vertex(33, (eyebrowRight * -5) +24.5);
  vertex(22, (eyebrowRight * -2) +8);
  vertex(18, -8);
  vertex(14, -17);
  vertex(17, -18);
  vertex(20, (eyebrowRight * -5) +14);
  endShape(CLOSE);

  line(10 + (eyebrowRight * 4), (eyebrowRight * -5) +12, 22, (eyebrowRight * -2) +8 );
  line(10 + (eyebrowRight * 4), (eyebrowRight * -5) +12, 25, -15 );
  line(25, -15, 4 + (eyebrowRight * 4), (eyebrowRight * -5) +13 );
  line(25, -15, 18, -8);
  line(33, (eyebrowRight * -5) +24.5, 23, -13);
  line(23, -13, 23, (eyebrowRight * -5) +17);
  line(23, (eyebrowRight * -5) +17, 20, (eyebrowRight * -5) +14);
  line(4 + (eyebrowRight * 4), (eyebrowRight * -5) +13, 23, (eyebrowRight * -5) +17);
  line(23, (eyebrowRight * -5) +17, 17, -18);
  line (17, -18, 18, -8);

  //eye left
  //  ellipse(-13, eyeLeft - 10, 4.5, -30+ eyeLeft*10); 
  beginShape();
  vertex(-13.5, 2+(eyeLeft*-3) );
  vertex(-12, eyeLeft-8 );
  vertex(-12.5, eyeLeft-7.5 );
  vertex(-15, eyeLeft-7.5 );
  vertex(-15, eyeLeft-9 );
  endShape(CLOSE);

  //  ellipse(-12, eyeLeft-8 , 2, 2); 

  //eye right
  //  ellipse(13, eyeRight -10, 4.5, -30 + eyeRight*10);
  beginShape();
  vertex(13.5, 2+(eyeRight*-3) );
  vertex(12, eyeRight-8 );
  vertex(12.5, eyeRight-7.5 );
  vertex(15, eyeRight-7.5 );
  vertex(15, eyeRight-9 );
  endShape(CLOSE);

  //mouth 
  //  rect(0, 20, mouthWidth, mouthHeight);


  //nose
  //  fill(38, 40, 40); 
  beginShape();
  vertex(2-mouthWidth/2, 20-mouthHeight/2);
  vertex(6-mouthWidth/2, 22.5-mouthHeight/2);
  vertex(7-mouthWidth/2, 22.5-mouthHeight/2);
  vertex(10-mouthWidth/2, 20-mouthHeight/2);
  vertex(12-mouthWidth/2, 14.5-mouthHeight/2);
  vertex(6-mouthWidth/2, 16.3-mouthHeight/2);
  vertex(0-mouthWidth/2, 14.5-mouthHeight/2);
  endShape(CLOSE);

  //ellipse(0-mouthWidth/2, 14.5-mouthHeight/2,5,5);


  //mouth
  //  fill(88, 30, 19); 
  beginShape();
  vertex(0-mouthWidth/2, 22-mouthHeight/2);
  vertex(6-mouthWidth/2, 22.5-mouthHeight/2);
  vertex(7-mouthWidth/2, 22.5-mouthHeight/2);
  vertex(0+mouthWidth/2, 22-mouthHeight/2);
  vertex(7-mouthWidth/2, 21+mouthHeight/2);
  vertex(5-mouthWidth/2, 21+mouthHeight/2);
  endShape(CLOSE);

  //top lip
  //  fill(133, 119, 110); 
  beginShape();
  vertex(0-mouthWidth/2, 22-mouthHeight/2);
  vertex(2-mouthWidth/2, 20-mouthHeight/2);
  vertex(6-mouthWidth/2, 22.5-mouthHeight/2);
  vertex(7-mouthWidth/2, 22.5-mouthHeight/2);
  vertex(10-mouthWidth/2, 20-mouthHeight/2);
  vertex(0+mouthWidth/2, 22-mouthHeight/2);
  vertex(7-mouthWidth/2, 22.5-mouthHeight/2);
  vertex(6-mouthWidth/2, 22.5-mouthHeight/2);
  endShape(CLOSE);


  //bottom lip
  //  fill(17, 16, 12); 
  beginShape();
  vertex(0-mouthWidth/2, 22-mouthHeight/2);
  vertex(5-mouthWidth/2, 21+mouthHeight/2);
  vertex(7-mouthWidth/2, 21+mouthHeight/2);
  vertex( 0+mouthWidth/2, 22-mouthHeight/2);
  vertex(8-mouthWidth/2, 22+mouthHeight/2);
  vertex(4-mouthWidth/2, 22+mouthHeight/2);
  endShape(CLOSE);


  //2
  //  fill(211, 204, 204); 
  beginShape();
  vertex(0-mouthWidth/2, 22-mouthHeight/2);
  vertex(4-mouthWidth/2, 22+mouthHeight/2);
  vertex(8-mouthWidth/2, 22+mouthHeight/2);
  vertex(0+mouthWidth/2, 22-mouthHeight/2);
  vertex(10-mouthWidth/2, 25+mouthHeight/2);
  vertex(7-mouthWidth/2, 35+mouthHeight/2);
  vertex(4-mouthWidth/2, 35+mouthHeight/2);
  vertex(2-mouthWidth/2, 25+mouthHeight/2);
  endShape(CLOSE);


  //1
  //  fill(80, 74, 69); 
  beginShape();
  vertex(4-mouthWidth/2, 22+mouthHeight/2);
  vertex(8-mouthWidth/2, 22+mouthHeight/2);
  vertex(10-mouthWidth/2, 25+mouthHeight/2);
  vertex(2-mouthWidth/2, 25+mouthHeight/2);
  endShape(CLOSE);


  //  //nostril left 
  //  ellipse(-5, nostrils * 1, nostrils, 3);
  //
  //  //nostril right 
  //  ellipse(5, nostrils * 1, nostrils, 3);
}


// OSC CALLBACK FUNCTIONS

public void found(int i) {
  println("found: " + i);
  found = i;
}

public void poseScale(float s) {
  println("scale: " + s);
  poseScale = s;
}

public void posePosition(float x, float y) {
  println("pose position\tX: " + x + " Y: " + y + " Y1: " + y1 );
  posePosition.set(x, y, 0);
}

public void poseOrientation(float x, float y, float z) {
  println("pose orientation\tX: " + x + " Y: " + y + " Z: " + z);
  poseOrientation.set(x, y, z);
}

public void mouthWidthReceived(float w) {
  println("mouth Width: " + w);
  mouthWidth = w;
}

public void mouthHeightReceived(float h) {
  println("mouth height: " + h);
  mouthHeight = h;
}

public void eyeLeftReceived(float f) {
  println("eye left: " + f);
  eyeLeft = f;
}

public void eyeRightReceived(float f) {
  println("eye right: " + f);
  eyeRight = f;
}

public void eyebrowLeftReceived(float f) {
  println("eyebrow left: " + f);
  eyebrowLeft = f;
}

public void eyebrowRightReceived(float f) {
  println("eyebrow right: " + f);
  eyebrowRight = f;
}

public void jawReceived(float f) {
  println("jaw: " + f);
  jaw = f;
}

public void nostrilsReceived(float f) {
  println("nostrils: " + f);
  nostrils = f;
}

// all other OSC messages end up here
void oscEvent(OscMessage m) {
  if (m.isPlugged() == false) {
    println("UNPLUGGED: " + m);
  }
}

Initially, I wanted to create a full-bodied, trippy-looking, fantastical creature whose motions reacted to the user’s facial expressions. I researched different character designs found on Pictoplasma and deviantART, but mostly drew inspiration from the Faun in Guillermo del Toro’s Pan’s Labyrinth. The Faun was a particularly interesting subject for me because not only was its design a well-crafted fusion of plant and animal, but also its movements and gestures strongly complemented its plantlike characteristics. My ‘final’ creature design, with its wooden ram horns, frog body, bushy hair, human hands, and venus flytrap for a tail, was inspired by the Faun and a bizarre legendary creature called the chimera.

Soon I realized that I was getting too entangled in the design and aesthetics that I did not give much thought into how the creature would move, or how it would interact with its environment. When I was unable to formulate an algorithm that would allow it to walk in a convincing manner, I decided to scrap my ram-frog-tree-bush-human-flytrap creature entirely and go with something more simple.

The ‘final‘ final creature design is a slightly cuter, llama-esque creature (because as I was painting it on Tuesday Jun noted that it resembled a llama). Its movements are controlled by the eyebrows, mouth, and head rotation – though I wanted it to be affected by the head tilt as well but ran into technical difficulties. I am more or less satisfied with the result, and found the extra ladybug feature to be an effective way to get the user more engaged with the character. If I had more time, I would make the ladybug fly more convincingly and try to work out the technical issues I had with the head tilting.

// a template for receiving face tracking osc messages from
// Kyle McDonald's FaceOSC https://github.com/kylemcdonald/ofxFaceTracker
//
// 2012 Dan Wilcox danomatika.com
// for the IACD Spring 2012 class at the CMU School of Art
//
// adapted from from Greg Borenstein's 2011 example
// http://www.gregborenstein.com/
// https://gist.github.com/1603230
//
import oscP5.*;
OscP5 oscP5;

// num faces found
int found;

// pose
float poseScale;
PVector posePosition = new PVector();
PVector poseOrientation = new PVector();

// gesture
float mouthHeight;
float mouthWidth;
float eyeLeft;
float eyeRight;
float eyebrowLeft;
float eyebrowRight;
float jaw;
float nostrils;

//fetus
PImage face;
PImage body;
PImage lefteye;
PImage righteye;
PImage leftlid;
PImage rightlid;
PImage upperlip;
PImage llamajaw;
PImage ladybug;
PImage flyinglady;

int x = 0;

boolean moving = false;
boolean flying = true;

Ladybug myLadybug;

void setup() {
  size(600, 525);

  face = loadImage("face.png");
  body = loadImage("body.png");
  lefteye = loadImage("left eye.png");
  righteye = loadImage("right eye.png");
  leftlid = loadImage("left lid.png");
  rightlid = loadImage("right lid.png");
  upperlip = loadImage("mouth top.png");
  llamajaw = loadImage("mouth bottom.png");
  ladybug = loadImage("ladybug.png");
  flyinglady = loadImage("ladybug-flying.png");

  oscP5 = new OscP5(this, 8338);
  oscP5.plug(this, "found", "/found");
  oscP5.plug(this, "poseScale", "/pose/scale");
  oscP5.plug(this, "posePosition", "/pose/position");
  oscP5.plug(this, "poseOrientation", "/pose/orientation");
  oscP5.plug(this, "mouthWidthReceived", "/gesture/mouth/width");
  oscP5.plug(this, "mouthHeightReceived", "/gesture/mouth/height");
  oscP5.plug(this, "eyeLeftReceived", "/gesture/eye/left");
  oscP5.plug(this, "eyeRightReceived", "/gesture/eye/right");
  oscP5.plug(this, "eyebrowLeftReceived", "/gesture/eyebrow/left");
  oscP5.plug(this, "eyebrowRightReceived", "/gesture/eyebrow/right");
  oscP5.plug(this, "jawReceived", "/gesture/jaw");
  oscP5.plug(this, "nostrilsReceived", "/gesture/nostrils");

  myLadybug = new Ladybug();

  noStroke();
  fill(0);
}

void draw() {  
  background(255);

  //I tried to allow the character to tilt its head
  //but due to high memory usage Processing just decided not to work. :/
  //translate(314,514);
  //rotate(poseOrientation.z);

  if (found > 0) {
    //pushMatrix();
    //translate(-314,-514);

    image(lefteye, 0, 0);
    image(righteye, 0, 0);

    float eyeDir = map(poseOrientation.y, -0.16, 0.04, -5, 5);
    ellipse(238+eyeDir, 337, 30, 30);
    ellipse(371+eyeDir, 337, 30, 30);

    /* for whatever reason, the eyebrow tracking is much more 
     appropriate for blink motions.... */
    float blinkleft = map(eyebrowLeft, 7.3, 8.350, -5, 45);
    float blinkright = map(eyebrowRight, 7.3, 8.350, -5, 45);

    image(leftlid, 0, blinkleft);
    image(rightlid, 0, blinkright);

    image(face, 0, 0);
    image(body, 0, 0);

    float mouthopen = map(mouthHeight, 1.0, 7.0, 0, 30);

    moving = mouthopen > 4.1;

    image(llamajaw, 0, mouthopen);

    image(upperlip, 0, 0);

    //popMatrix();
  }

  myLadybug.fly();
  myLadybug.display();
}

// OSC CALLBACK FUNCTIONS

public void found(int i) {
  println("found: " + i);
  found = i;
}

public void poseScale(float s) {
  println("scale: " + s);
  poseScale = s;
}

public void posePosition(float x, float y) {
  println("pose position\tX: " + x + " Y: " + y );
  posePosition.set(x, y, 0);
}

public void poseOrientation(float x, float y, float z) {
  println("pose orientation\tX: " + x + " Y: " + y + " Z: " + z);
  poseOrientation.set(x, y, z);
}

public void mouthWidthReceived(float w) {
  println("mouth Width: " + w);
  mouthWidth = w;
}

public void mouthHeightReceived(float h) {
  println("mouth height: " + h);
  mouthHeight = h;
}

public void eyeLeftReceived(float f) {
  println("eye left: " + f);
  eyeLeft = f;
}

public void eyeRightReceived(float f) {
  println("eye right: " + f);
  eyeRight = f;
}

public void eyebrowLeftReceived(float f) {
  println("eyebrow left: " + f);
  eyebrowLeft = f;
}

public void eyebrowRightReceived(float f) {
  println("eyebrow right: " + f);
  eyebrowRight = f;
}

public void jawReceived(float f) {
  println("jaw: " + f);
  jaw = f;
}

public void nostrilsReceived(float f) {
  println("nostrils: " + f);
  nostrils = f;
}

// all other OSC messages end up here
void oscEvent(OscMessage m) {
  if (m.isPlugged() == false) {
    println("UNPLUGGED: " + m);
  }
}

class Ladybug {
  float posX;
  float posY;
  float spdX;
  float spdY;

  Ladybug() {
    posX = random(40, 400);
    posY = random(60, 400);
    spdX = random(-2, 2);
    spdY = random(-2, 2);
  }

  void fly() {
    if (!moving && 264 < = posX && posX <= 303 
      && 300 <= posY && posY <= 342) {
      flying = false;
      spdX = 0;
      spdY = 0;
    }

    else {
      if (spdX == 0) {
        //it'll move away from the mouth
        spdX = random(-2, 2);
        spdY = random(-3, 0);
      }

      posX += spdX;
      posY += spdY;
      flying = true;

      if (posX < 40 || posX > 400) {
        spdX *= -1;
      }

      if (posY < 60 || posY > 400) {
        spdY *= -1;
      }
    }
  }

  void display() {

    if (flying) {
      image(flyinglady, posX, posY);
    }

    else {
      image(ladybug, posX, posY);
    }
  }
} 

< \pre>