cambu-mocap
The Story
When I was about 12, I visited the North American veterinary conference (NAVC) with my mom in Orlando, Florida. I was walking around the show floor with my mom when we decided to stop at the Bayer booth. In the middle of the booth was an original Microsoft Surface table — many people were congregating around to to see what it was all about. My mom and I played with it for awhile and then she left to enjoy the rest of the conference, but I stayed in the Bayer booth for easily 3 or 4 more hours becoming good friends with the booth attendants. I think it was the first highly responsive touch interface I’d ever used and it played on in my dreams for weeks. When I returned home, I tried to get my dad to buy one for our house, but at the time it was ~10-15K to install and you had to be a commercial partner…
Documentation
Code
//include statements for the library
import oscP5.*;
import netP5.*;
img image1; //Constructor for Image
hand leftHand; //the object that will contain all of the leftHand Data
hand rightHand; //the object that will contain all of the rightHand Data
OscP5 oscP5; //name the oscP5 object
NetAddress serverAddress; //name the addresses you'll send and receive @
PImage imageFill1;
int listeningPort; //server and client ports
float rectX = 200;
float rectY = 200;
float rectWidth = 350;
float rectHeight = 250;
//now set the addresses, etc
void setup()
{
imageFill1 = loadImage("IMG_1087.JPG");
//if listening and sending are the same then messages will be sent back to this sketch
listeningPort = 12345;
oscP5 = new OscP5(this, listeningPort);
size(1200, 700);
background(rectX, rectY, rectWidth, rectHeight);
// create image object
image1 = new img(rectX, rectY, rectWidth, rectHeight);
// create hand objects
leftHand = new hand();
rightHand = new hand();
}
void oscEvent(OscMessage receivedMessage) {
String[] message = receivedMessage.addrPattern().split("/");
//ripping out all joint:hand data
boolean isHand = message[4].equals("HandLeft") || message[4].equals("HandRight");
if (message[3].equals("joints") && isHand == true) {
if (message[4].equals("HandLeft")) {
float handLeftXPos = receivedMessage.get(0).floatValue();
float handLeftYPos = receivedMessage.get(1).floatValue();
String tracked = receivedMessage.get(3).stringValue();
leftHand.updateXYC(handLeftXPos, handLeftYPos, tracked);
}
if (message[4].equals("HandRight")) {
float handRightXPos = receivedMessage.get(0).floatValue();
float handRightYPos = receivedMessage.get(1).floatValue();
String tracked = receivedMessage.get(3).stringValue();
rightHand.updateXYC(handRightXPos, handRightYPos, tracked);
}
}
//ripping out all hand:closed data
if (message[3].equals("hands")) {
String leftOrRight = message[4];
String grabVar = (receivedMessage.get(0).stringValue() + "/" + leftOrRight);
if (grabVar.contains("Left")) {//change something about left
if (grabVar.contains("Open")) {
leftHand.updateIsClosed(false);
} else {
leftHand.updateIsClosed(true);
}
}
if (grabVar.contains("Right")) {//change something about the right hand
if (grabVar.contains("Open")) {
rightHand.updateIsClosed(false);
} else {
rightHand.updateIsClosed(true);
}
}
}
//println ("rectX" + rectX);
//println ("rectY" + rectY);
//println ("rectWidth" + rectWidth);
//println ("rectHeight" + rectHeight);
}
void hoverCheck() {
//check if right hand is hovering over the object
if (rightHand.xPos >= image1.xPosition && rightHand.xPos <= image1.xPosition + image1.rectWidth && rightHand.yPos >= image1.yPosition && rightHand.yPos <= image1.yPosition + image1.rectHeight) { //println(rightHand.xPos + " >= " + rectX + " && " + rightHand.xPos + " < = " + (rectX+rectWidth)); image1.updateHoverState(true); if (rightHand.closed == true) { println("hoverGrab"); image1.move(rightHand.xPos, rightHand.yPos); toScale(); } } else { image1.updateHoverState(false); } } void toScale() { if (leftHand.xPos >= image1.xPosition && leftHand.xPos <= image1.xPosition + image1.rectWidth && leftHand.yPos >= image1.yPosition && leftHand.yPos <= image1.yPosition + image1.rectHeight) {
//left hand also hovering
if (leftHand.closed == true) {
//get distance
float rightToLeftDist = dist(rightHand.xPos, rightHand.yPos, leftHand.xPos,leftHand.yPos);
println(rightToLeftDist);
float scaleVar = map(rightToLeftDist, 0, 0.5*image1.rectWidth, 0, 1.5);
image1.rectWidth = image1.rectWidth*scaleVar;
image1.rectHeight = image1.rectHeight*scaleVar;
//scale by some multuplier
}
}
}
void draw() {
noStroke();
fill(255, 255, 255, 100);
rect(0, 0, width, height);
hoverCheck();
//image1.render();
image(imageFill1, image1.xPosition, image1.yPosition);
imageFill1.resize(int(image1.rectWidth), int(image1.rectHeight));
image1.render();
scale(1);
leftHand.render();
rightHand.render();
}
class hand { //class that allows the creation of any hand method
boolean closed;
float xPos;
float yPos;
color fillColor;
String trackingConfidence; //is either Tracked, Inferred, or (maybe something else)
hand() {
closed = false;
xPos = 200;
yPos = 200;
fillColor = color(200, 200, 200);
}
void updateXYC(float newXPos, float newYPos, String trackedState) { // a function to update x position, y position, and tracking confidence
//direct map
//xPos = map(newXPos, -1, 1, 0, width);
//yPos = map(newYPos, 1, -1, 0, height);
//smooothed map
//X------
float mappedNewXPos = map(newXPos, -1, 1, 0, width);
//println(mappedNewXPos);
xPos = 0.5 * xPos + 0.5 * mappedNewXPos;
//Y------
float mappedNewYPos = map(newYPos, 1, -1, 0, height);
//println(mappedNewXPos + "," + mappedNewYPos);
yPos = 0.5 * yPos + 0.5 * mappedNewYPos;
trackingConfidence = trackedState;
}
void updateIsClosed(boolean openOrClose) {
if (openOrClose == true) {
fillColor = color(230, 50, 100);
closed = true;
} else { // open
fillColor = color(200, 200, 200);
closed = false;
}
}
void render() {
fill(fillColor);
ellipse(xPos, yPos, 25, 25);
}
}
class img {
color c;
float xPosition;
float yPosition;
float rectWidth;
float rectHeight;
boolean isHovering;
img(float xPos, float yPos, float rWidth, float rHeight) {
c = color(200, 200, 200, 0);
xPosition = xPos;
yPosition = yPos;
rectWidth = rWidth;
rectHeight = rHeight;
isHovering = false;
}
void render() {
fill(c);
rect(xPosition, yPosition, rectWidth, rectHeight);
}
void updateHoverState(boolean hoverState) {
isHovering = hoverState;
if (isHovering) {
c = color(245, 50, 100, 50);
} else {
c = color(245, 50, 100, 0);
}
}
void move(float x, float y) {
//xPosition = xPosition + deltaX;
//yPosition = yPosition + deltaY;
xPosition = x-rectWidth/2;
yPosition = y-rectHeight/2;
}
}