FaceOSC
Defense Mechanisms
My initial idea was to create an onscreen character which takes a neutral/unfriendly expression and exaggerates it, literally reflecting the person’s “prickliness” or unapproachability. When a person smiles, then the character becomes rounded and happier.
In the end, while I worked on the sketch, I modified the concept. It became more of a creature and less of a puppet. The character becomes rounder and more visible when you smile, and pricklier and less visible for the longer you frown. It also moves away toward the corner of the window for the longer you frown. Altogether, the character reacts to your expressions (or reflects your emotions, depending how you interpret it) by becoming more or less defensive (reflected in visibility, proximity, and prickliness.)
If I had more time to spend on this sketch, I would have experimented with moving it back toward my original concept—adding back facial features and making this more like a puppet.
Code
pricklyFace (main)
import oscP5.*;
OscP5 oscP5;
// our FaceOSC tracked face dat
Face face = new Face();
float faceScale = 1; // default - no resizing of face
ArrayList faceOutline = new ArrayList ();
int numPoints = 100;
float initialPrickliness = 0.2;
float prickliness = initialPrickliness;
float maxPrickliness = 0.7;
float minPrickliness = 0;
float closeness = 0.3;
float maxCloseness = 0.7;
float minCloseness = 0.15;
void setup() {
// default size is 640 by 480
int defaultWidth = 640;
int defaultHeight = 480;
faceScale = 1; // shrink by half
int realWidth = (int)(defaultWidth * faceScale);
int realHeight = (int)(defaultHeight * faceScale);
size(realWidth, realHeight, OPENGL);
frameRate(10);
oscP5 = new OscP5(this, 8338);
}
void draw() {
background(250);
noStroke();
updatePrickliness();
if (face.found > 0) {
// draw such that the center of the face is at 0,0
translate(face.posePosition.x*faceScale*closeness, face.posePosition.y*faceScale*closeness);
// scale things down to the size of the tracked face
// then shrink again by half for convenience
closeness = map(prickliness, minPrickliness, maxPrickliness, maxCloseness, minCloseness);
scale(face.poseScale*closeness);
// rotate the drawing based on the orientation of the face
rotateY (0 - face.poseOrientation.y);
rotateX (0 - face.poseOrientation.x);
// rotateZ ( face.poseOrientation.z);
float fill = map(prickliness, minPrickliness, maxPrickliness, 100, 200);
fill((int)fill);
// drawEyes();
// drawMouth();
// print(face.toString());
faceOutline = new ArrayList ();
getFaceOutlinePoints();
drawOutline();
/*if (face.isBlinking()) {
println("BLINKED");
}
face.lastEyeHeight = face.eyeLeft;
face.lastEyebrowHeight = face.eyeRight;
*/
}
}
// OSC CALLBACK FUNCTIONS
void oscEvent(OscMessage m) {
face.parseOSC(m);
}
void drawOutline() {
float x = 0;
float y = 0;
if (faceOutline.size() != (numPoints + 1)) {
getFaceOutlinePoints();
return;
}
else {
beginShape();
for (int i=0; i < = numPoints; i++) {
x = faceOutline.get(i).x;
y = faceOutline.get(i).y;
vertex(x, y);
}
endShape();
}
}
void updatePrickliness() {
float antiPrickliness = 0;
int transitionTime = 30000;
if (!face.isSmiling()) {
prickliness = constrain(face.timeSinceSmile, 0, transitionTime);
prickliness = map(prickliness, 0, transitionTime, minPrickliness, maxPrickliness);
}
antiPrickliness = constrain(face.smilingTime, 0, transitionTime);
antiPrickliness = -1 * map(antiPrickliness, 0, transitionTime, minPrickliness, maxPrickliness);
prickliness = prickliness + antiPrickliness;
constrain(prickliness, minPrickliness, maxPrickliness);
if (prickliness < 0) {
prickliness = 0;
}
}
void getFaceOutlinePoints() {
int xCenter = 0;
int yCenter = 0;
for (int i=0; i <= numPoints; i++) {
float radius = 30;
// iterate and draw points around circle
float theta = 0;
float x;
float y;
float oldRadius = -1;
theta = map(i, 0, numPoints, 0, 2*PI);
if (i%2 == 0) {
oldRadius = radius;
radius = radius * random(1+prickliness, 1+(prickliness*2));
}
x = radius*cos(theta) + xCenter;
y = radius*sin(theta) + yCenter;
if (i == numPoints +1) {
PVector firstPoint = faceOutline.get(0);
PVector circlePoint = new PVector(firstPoint.x, firstPoint.y);
faceOutline.add(circlePoint);
}
else {
PVector circlePoint = new PVector(x, y);
faceOutline.add(circlePoint);
}
if (oldRadius > 0) {
radius = oldRadius;
oldRadius = -1;
}
}
}
void drawEyes() {
int distanceFromCenterOfFace = 14;
int heightOnFace = -4;
int eyeWidth = 6;
int eyeHeight = 4;
ellipse(-1*distanceFromCenterOfFace, face.eyeLeft * heightOnFace, eyeWidth, eyeHeight);
ellipse(distanceFromCenterOfFace, face.eyeRight * heightOnFace, eyeWidth, eyeHeight);
}
void drawMouth() {
float mouthWidth = 30;
int heightOnFace = 14;
int mouthHeightFactor = 3;
float mLeftCornerX = 0;
float mLeftCornerY = heightOnFace;
float pointX = mLeftCornerX + ((mouthWidth/2));
float mouthHeight = face.mouthHeight * mouthHeightFactor;
ellipse(mLeftCornerX, mLeftCornerY, mouthWidth, mouthHeight);
}
Face class
import oscP5.*;
// a single tracked face from FaceOSC
class Face {
// num faces found
int found;
// pose
float poseScale;
PVector posePosition = new PVector();
PVector poseOrientation = new PVector();
// gesture
float mouthHeight, mouthWidth;
float eyeLeft, eyeRight;
float eyebrowLeft, eyebrowRight;
float jaw;
float nostrils;
// past
float lastEyeHeight;
float lastEyebrowHeight;
boolean wasSmiling = false;
float startedSmilingTime = 0;
float smilingTime = 0;
float stoppedSmilingTime = 0;
float timeSinceSmile = 10000;
Face() {
}
boolean isSmiling() {
if (mouthIsSmiling()) {
if (wasSmiling == false) {
wasSmiling = true;
startedSmilingTime = millis();
timeSinceSmile = 0;
}
else {
smilingTime = millis() - startedSmilingTime;
println("smilingTime: ");
print(smilingTime);
println("");
}
return true;
}
else {
if (wasSmiling == false) {
timeSinceSmile = millis() - stoppedSmilingTime;
}
else {
wasSmiling = false;
stoppedSmilingTime = millis();
smilingTime = 0;
}
return false;
}
}
boolean mouthIsSmiling() {
float minSmileWidth = 15;
float minSmileHeight = 2;
return ((mouthWidth > minSmileWidth) && (mouthHeight > minSmileHeight));
}
boolean isBlinking() {
float eyeHeight = (face.eyeLeft + face.eyeRight) / 2;
float eyebrowHeight = (face.eyebrowLeft + face.eyebrowRight) / 2;
if ((eyeHeight < lastEyeHeight) &&
(eyebrowHeight > lastEyebrowHeight)) {
return true;
}
return false;
}
boolean isSpeaking() {
int speakingMouthHeightThreshold = 2;
if (face.mouthHeight > speakingMouthHeightThreshold) {
return true;
}
else {
return false;
}
}
// parse an OSC message from FaceOSC
// returns true if a message was handled
boolean parseOSC(OscMessage m) {
if (m.checkAddrPattern("/found")) {
found = m.get(0).intValue();
return true;
}
// pose
else if (m.checkAddrPattern("/pose/scale")) {
poseScale = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/pose/position")) {
posePosition.x = m.get(0).floatValue();
posePosition.y = m.get(1).floatValue();
return true;
}
else if (m.checkAddrPattern("/pose/orientation")) {
poseOrientation.x = m.get(0).floatValue();
poseOrientation.y = m.get(1).floatValue();
poseOrientation.z = m.get(2).floatValue();
return true;
}
// gesture
else if (m.checkAddrPattern("/gesture/mouth/width")) {
mouthWidth = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/mouth/height")) {
mouthHeight = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/eye/left")) {
eyeLeft = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/eye/right")) {
eyeRight = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/eyebrow/left")) {
eyebrowLeft = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/eyebrow/right")) {
eyebrowRight = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/jaw")) {
jaw = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/nostrils")) {
nostrils = m.get(0).floatValue();
return true;
}
return false;
}
// get the current face values as a string (includes end lines)
String toString() {
return "found: " + found + "\n"
+ "pose" + "\n"
+ " scale: " + poseScale + "\n"
+ " position: " + posePosition.toString() + "\n"
+ " orientation: " + poseOrientation.toString() + "\n"
+ "gesture" + "\n"
+ " mouth: " + mouthWidth + " " + mouthHeight + "\n"
+ " eye: " + eyeLeft + " " + eyeRight + "\n"
+ " eyebrow: " + eyebrowLeft + " " + eyebrowRight + "\n"
+ " jaw: " + jaw + "\n"
+ " nostrils: " + nostrils + "\n";
}
};
Being Shushed
My second idea was to create a character and, to some degree, an environment/game mechanic. When you open your mouth, a small speech bubble appears and begins to grow. However, as soon as you open your mouth, the word “shhhhh” begins to appear all around, and the words cluster around the speech bubble, as though they are squishing it. If you close your mouth, the speech bubble disappears and the face onscreen looks somewhat unhappy. But if you keep your mouth open long enough, the bubble grows and pushes the shhh’es out of the frame. If you successfully do this, you see the word applause appear all around.
I attempted to implement this idea and part of the way. I created (as shown in the video above) a speech bubble which grows based on how long you have been “speaking” (crudely measured by the length of time which you have had your mouth open). However, I had trouble figuring out how to position the face and speech bubble on screen such that they wouldn’t overlap awkwardly. I also realized that implementing some sort of particle system (most likely) of “shhh”es to put pressure on the speech bubble was going to make realizing this fully take a ton more time.
If I had more time to spend on this, I would probably stop drawing the face temporarily and work on the speech bubble’s interaction with a particle system of “shhh”es, then come back to the issue of the speaker’s face.
Code
shhhFace
//
// a template for receiving face tracking osc messages from
// Kyle McDonald's FaceOSC https://github.com/kylemcdonald/ofxFaceTracker
//
// this example includes a class to abstract the Face data
//
// 2012 Dan Wilcox danomatika.com
// for the IACD Spring 2012 class at the CMU School of Art
//
// adapted from from Greg Borenstein's 2011 example
// http://www.gregborenstein.com/
// https://gist.github.com/1603230
import oscP5.*;
OscP5 oscP5;
// our FaceOSC tracked face dat
Face face = new Face();
SpeechBubble speechBubble = new SpeechBubble();
float faceScale = 1;
// for additions
void setup() {
// default size is 640 by 480
int defaultWidth = 640;
int defaultHeight = 480;
int realWidth = (int)(defaultWidth * faceScale);
int realHeight = (int)(defaultHeight * faceScale);
size(realWidth, realHeight, OPENGL);
frameRate(30);
oscP5 = new OscP5(this, 8338);
}
void draw() {
background(255);
stroke(0);
if (face.found > 0) {
// draw such that the center of the face is at 0,0
translate(face.posePosition.x*faceScale, face.posePosition.y*faceScale);
// scale things down to the size of the tracked face
// then shrink again by half for convenience
scale(face.poseScale*0.5);
// rotate the drawing based on the orientation of the face
rotateY (0 - face.poseOrientation.y);
rotateX (0 - face.poseOrientation.x);
rotateZ ( face.poseOrientation.z);
noFill();
drawEyes();
drawMouth();
face.isSpeaking();
int sbX = 7;
int sbY = -15;
speechBubble.draw(sbX, sbY);
//}
//drawNose();
//drawEyebrows();
print(face.toString());
if (face.isSmiling()) {
println("SMILING");
}
if (face.isBlinking()) {
println("BLINKED");
}
face.lastEyeHeight = face.eyeLeft;
face.lastEyebrowHeight = face.eyeRight;
println("lastEyeHeight " + face.lastEyeHeight);
println("lastEyebrowHeight " + face.lastEyebrowHeight);
}
}
// OSC CALLBACK FUNCTIONS
void oscEvent(OscMessage m) {
face.parseOSC(m);
}
void drawEyes() {
int distanceFromCenterOfFace = 20;
int heightOnFace = -9;
int eyeWidth = 6;
int eyeHeight =5;
ellipse(-1*distanceFromCenterOfFace, face.eyeLeft * heightOnFace, eyeWidth, eyeHeight);
ellipse(distanceFromCenterOfFace, face.eyeRight * heightOnFace, eyeWidth, eyeHeight);
}
void drawEyebrows() {
rectMode(CENTER);
fill(0);
int distanceFromCenterOfFace = 20;
int heightOnFace = -5;
int eyebrowWidth = 23;
int eyebrowHeight = 2;
rect(-1*distanceFromCenterOfFace, face.eyebrowLeft * heightOnFace, eyebrowWidth, eyebrowHeight);
rect(distanceFromCenterOfFace, face.eyebrowRight * heightOnFace, eyebrowWidth, eyebrowHeight);
}
void drawMouth() {
float mouthWidth = 30;
int heightOnFace = 14;
int mouthHeightFactor = 6;
float mLeftCornerX = 0;
float mLeftCornerY = heightOnFace;
float pointX = mLeftCornerX + ((mouthWidth/2));
float mouthHeight = face.mouthHeight * mouthHeightFactor;
ellipse(mLeftCornerX, mLeftCornerY, mouthWidth, mouthHeight);
}
void drawNose() {
int distanceFromCenterOfFace = 5;
int heightOnFace = -1;
int nostrilWidth = 4;
int nostrilHeight = 3;
ellipse(-1*distanceFromCenterOfFace, face.nostrils * heightOnFace, nostrilWidth, nostrilHeight);
ellipse(distanceFromCenterOfFace, face.nostrils * heightOnFace, nostrilWidth, nostrilHeight);
}
SpeechBubble
class SpeechBubble {
float xPos;
float yPos;
float sbHeight = 150*0.25;
float sbWidth = 250*0.25;
float initialRadius = (sbHeight/3);
float radius = initialRadius;
int numPoints = 30;
// http://math.rice.edu/~pcmi/sphere/degrad.gif
float extrusionTheta = (5*PI)/6;
float epsilon = PI/25;
void draw(float xPosition, float yPosition) {
xPos = xPosition;
yPos = yPosition;
float timeRadiusFactor = face.totalTime/10000;
radius = radius + timeRadiusFactor;
if (radius < 10) {
return;
}
float xCenter = xPos+sbWidth/2 + timeRadiusFactor;
float yCenter = yPos+sbHeight/2 - (timeRadiusFactor/2);
println("DRAWN");
beginShape();
// variables for calculating each point
float x;
float y;
float theta;
// iterate and draw points around circle.
for (int i = 0; i <= numPoints; i++) {
theta = map(i, 0, numPoints-2, 0, 2*PI);
// this minus-2 is a hack to make the circle close
x = radius*cos(theta) + xCenter;
y = radius*sin(theta) + yCenter;
// check to see if we're at the point in the circle where
// we want to draw the part of the speech bubble that sticks out
if (((theta - epsilon) < extrusionTheta) &&
((theta + epsilon) > extrusionTheta)){
float extrusionRadius = PI/25;
float startTheta = extrusionTheta - extrusionRadius;
float endTheta = extrusionTheta + extrusionRadius;
float startX = radius*cos(startTheta) + xCenter;
float startY = radius*sin(startTheta) + yCenter;
float endX = radius*cos(endTheta) + xCenter;
float endY = radius*sin(endTheta) + yCenter;
curveVertex(startX, startY);
vertex(startX, startY);
vertex(x - (radius/1.5), y+ (radius/3));
vertex(endX, endY);
curveVertex(endX, endY);
}
else {
curveVertex(x, y);
}
}
endShape();
}
}
Face class
import oscP5.*;
// a single tracked face from FaceOSC
class Face {
// num faces found
int found;
// pose
float poseScale;
PVector posePosition = new PVector();
PVector poseOrientation = new PVector();
// gesture
float mouthHeight, mouthWidth;
float eyeLeft, eyeRight;
float eyebrowLeft, eyebrowRight;
float jaw;
float nostrils;
// past
float lastEyeHeight;
float lastEyebrowHeight;
boolean wasSpeaking = false;
float startSpeakingTime = 0;
float totalTime = 0;
float stoppedSpeakingTime = 0;
Face() {
}
boolean isSmiling() {
float minSmileWidth = 15;
float minSmileHeight = 2;
if ((mouthWidth > minSmileWidth) &&
(mouthHeight > minSmileHeight)) {
return true;
}
return false;
}
boolean isBlinking() {
float eyeHeight = (face.eyeLeft + face.eyeRight) / 2;
float eyebrowHeight = (face.eyebrowLeft + face.eyebrowRight) / 2;
if ((eyeHeight < lastEyeHeight) &&
(eyebrowHeight > lastEyebrowHeight)) {
return true;
}
return false;
}
boolean isSpeaking() {
int speakingMouthHeightThreshold = 2;
/* Debug:
println("MOUTHHEIGHT");
println(face.mouthHeight);
*/
println("totalTime: ");
print(totalTime);
println("");
if (face.mouthHeight > speakingMouthHeightThreshold) {
if (!wasSpeaking) {
totalTime = 0;
startSpeakingTime = millis();
wasSpeaking = true;
}
else {
totalTime = millis() - startSpeakingTime;
}
println("SPEAKING");
return true;
}
else {
if (wasSpeaking) {
println("NOT SPEAKING");
stoppedSpeakingTime = millis();
wasSpeaking = false;
totalTime = 0;
}
else {
totalTime = -1*(millis() - stoppedSpeakingTime);
}
return false;
}
}
// parse an OSC message from FaceOSC
// returns true if a message was handled
boolean parseOSC(OscMessage m) {
if (m.checkAddrPattern("/found")) {
found = m.get(0).intValue();
return true;
}
// pose
else if (m.checkAddrPattern("/pose/scale")) {
poseScale = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/pose/position")) {
posePosition.x = m.get(0).floatValue();
posePosition.y = m.get(1).floatValue();
return true;
}
else if (m.checkAddrPattern("/pose/orientation")) {
poseOrientation.x = m.get(0).floatValue();
poseOrientation.y = m.get(1).floatValue();
poseOrientation.z = m.get(2).floatValue();
return true;
}
// gesture
else if (m.checkAddrPattern("/gesture/mouth/width")) {
mouthWidth = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/mouth/height")) {
mouthHeight = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/eye/left")) {
eyeLeft = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/eye/right")) {
eyeRight = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/eyebrow/left")) {
eyebrowLeft = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/eyebrow/right")) {
eyebrowRight = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/jaw")) {
jaw = m.get(0).floatValue();
return true;
}
else if (m.checkAddrPattern("/gesture/nostrils")) {
nostrils = m.get(0).floatValue();
return true;
}
return false;
}
// get the current face values as a string (includes end lines)
String toString() {
return "found: " + found + "\n"
+ "pose" + "\n"
+ " scale: " + poseScale + "\n"
+ " position: " + posePosition.toString() + "\n"
+ " orientation: " + poseOrientation.toString() + "\n"
+ "gesture" + "\n"
+ " mouth: " + mouthWidth + " " + mouthHeight + "\n"
+ " eye: " + eyeLeft + " " + eyeRight + "\n"
+ " eyebrow: " + eyebrowLeft + " " + eyebrowRight + "\n"
+ " jaw: " + jaw + "\n"
+ " nostrils: " + nostrils + "\n";
}
};
Other Idea: Feeling Misinterpreted
One of my initial ideas was to create a face/character that would mirror your expressions but be…off. The face itself would be distorted, somewhat ugly, with some features upside-down or asymmetrical. As you looked at the face, it would mirror your expressions somewhat—if you smile, it would smile too, but crookedly, awkwardly.
The concept for this was to create a sort of mirror that evokes the feeling of being misinterpreted, not being able to say the right thing, or express it effectively.
I abandoned this idea because after initial experimentation, I decided that it would be too difficult to get to the point of accurately mirroring a face’s expressions so that I could deliberately distort parts of that mirroring.