Category: 05-Body

Sepho Body

For my body project I created a program that tracks the users face in order to get their distance from the screen. The screen displays a map of Pittsburgh and the closer to the screen you move your face the more the map zooms in. It will continue to zoom in until your face is almost touching the screen at which point it will show you a view of yourself from above. For me this piece was meant to be a commentary of the state of surveillance through things such as satellites, facial recognition software and personal devices in general.

https://vimeo.com/306442105

 

rigatoni-Body

Head-Games
Instructions
Move your nose left and right to roll Bobo
Jerk your nose up to make Bobo jump for dear life!

Originally my plan was to make a simplified version of a game like Stick Fight or Super Smash Bros, but allow players to handle movement with their head via webcam, freeing up both their hands for the combat aspect of those games which can be complex. However I was having a lot of trouble getting the networking, matter.js and posenet components working together so I decided to boil the concept down to its most basic unique element, which was the movement.

I have noticed that when people play highly-movement centric games like platformers and racing games that they almost involuntarily jerk their body toward where they want their avatar to be. Its amusing to watch especially non-gamers frantically hopping around in their seats as they get used to the controls of a new game. I thought it would be interesting to have this kind of physical response to platformers be an actual element of its control rather than just a by-product.

My main challenge here was making the head controls comfortable. In an earlier iteration of this game I noticed the back of my neck was getting sore after playing it for more than a minute or so. Most of my changes after that were trying to find the right balance of values for tracking jumps, and I feel like I need to add sensitivity controls because the few people I tested this with had widely different ways of getting their character to jump, some being far more forceful than others. I also wish I had given myself more time to document this work and record a full demo so I could have made use of the in-class critiques.

In conclusion, I think I will be making use of posenet in future projects. In addition, I enjoyed working with matter.js, it was my first time using it and I don't think I even scraped the surface of what was possible, and I hope to do that as well in the future.

var Engine = Matter.Engine,
    Render = Matter.Render,
    World = Matter.World,
    Bodies = Matter.Bodies;
var engine = Engine.create();
var render = Render.create({
    element: document.body,
    engine: engine,
    options: {width:800, height:800,
             pixelRatio:1,
             wireframes:false}
});

Engine.run(engine);
Render.run(render);

let platTex = "https://cdn.glitch.com/7541f658-c2e5-4490-8bac-21a2d3c09449%2FtestPlatformTex.jpg?1539315376974";
let bobo1Tex = "https://cdn.glitch.com/7541f658-c2e5-4490-8bac-21a2d3c09449%2FgroundBobo.png?1539318024497";
let bobo2Tex = "https://cdn.glitch.com/7541f658-c2e5-4490-8bac-21a2d3c09449%2FjumpBobo.png?1539318026058";

var player;
var poseNet;
var platforms = [];

function setup() {
  createCanvas(800, 800);
  video = createCapture(VIDEO);
  video.size(width, height);
  poseNet = new PoseNetObj(video);
  poseNet.poseNet.on('pose', function (results) {
    poseNet.poses = results;
  });
  Reset();
}

function draw() {
  image(video, 0, 0, 800, 800); 
}

function Reset() {
  World.clear(engine.world);
  Engine.clear(engine);
  engine.events = {};
  player = new Player(poseNet);
  platforms.push(new Platform(200, 300, 1500, 20));
}

function GameLoop() {
  player.Update(); 
  for(var i=0; i-10 || abs(this.gameObject.velocity.y)>2) {
        relativeHeadPos.y=0;
      } else {
        relativeHeadPos.y=-2000;
      }
        
      if(abs(this.gameObject.velocity.y)>1) {
        relativeHeadPos.x/=4; 
        this.gameObject.render.sprite.texture = bobo2Tex;
      } else {
        this.gameObject.render.sprite.texture = bobo1Tex; 
      }
      this.velocity.x = relativeHeadPos.x/this.inertia;
      this.velocity.y = relativeHeadPos.y/this.inertia;
      this.prevY = this.poseSource.GetHeadPos().y;

      Matter.Body.applyForce(this.gameObject, this.gameObject.position, this.velocity);
    } 
  }
  
  this.CheckBounds = function() {
    if(this.gameObject.position.x<-10 || this.gameobject.position.x>1500) {
      console.log("game over");
      return true; 
    }
    if(this.gameObject.position.y<-100 || this.gameobject.position.y> 1000) {
      console.log("game over");
      return true; 
    }
    return false;
  }
}

function PoseNetObj(videoSource) {
  this.video = videoSource;
  this.poses = [];
  this.poseNet = ml5.poseNet(video, {flipHorizontal:true, detectionType:'single'});
  video.hide();
  this.lastKnownPos = {x:width/2, y:height/2};
  
  this.GetHeadPos = function() {
    let playerPose = this.poses[0]; 
    if(typeof(playerPose)!="undefined") {
      this.lastKnownPos.x = playerPose.pose.keypoints[0].position.x;
      this.lastKnownPos.y = playerPose.pose.keypoints[0].position.y;
    } 
    return this.lastKnownPos;
  }
}

maya-Body

I liked the contrast between skin color and mouth one very much and wanted to make a work using it. Then, I remembered the proverb "Out of the mouth comes evil". This proverb is used in the same meaning as "Be careful what you say." in Japan. (Everyone knows this proverb in Japan, but I heard  that people in the US are not familiar with it, so I explained it.) In this work, I expressed the trouble that happens when you speak. The trigger that will bring trouble is the action opening your mouth, and the effect of screen break is used the algorithm of Voronoi.

Movie

 

Sketch

 

Code

var ctracker;
var videoInput;
var VideoGraphics, Graphics, FinalGraphics;
var shader, program, randomPoints, fade;
var startTime = 0;
var openMouth = false;
 
function setup() {
 
    deb = Date.now();
 
    // setup camera capture
    videoInput = createCapture();
    videoInput.size(800, 600);
    videoInput.hide();
    VideoGraphics = createGraphics(600, 600);
 
 
    // setup canvas
    var cnv = createCanvas(800, 600);
    cnv.position(0, 0);
 
    // setup tracker
    ctracker = new clm.tracker();
    ctracker.init(pModel);
    ctracker.start(videoInput.elt);
 
 
    noStroke();
    Graphics = createGraphics(800, 600);
    FinalGraphics = createGraphics(600, 600);
 
 
    //shader
    shader = createGraphics(600, 600, WEBGL);
    program = shader.createShader(vert, frag);
    shader.noStroke();
    //voronoi
    randomPoints = createGraphics(30, 30);
    randomPoints.noStroke();
    randomPoints.fill(255, 0, 0);
    randomPoints.rect(0, 0, randomPoints.width, randomPoints.height);
 
    randomPoints.loadPixels();
    var d = randomPoints.pixelDensity();
    var size = 4 * (randomPoints.width * d) * (randomPoints.height * d);
    for (var i = 0; i &lt; size; i += 4) {
        randomPoints.pixels[i] = random(0, 255);
        randomPoints.pixels[i + 1] = random(0, 255);
        randomPoints.pixels[i + 2] = 0;//blue(c);
        randomPoints.pixels[i + 3] = 255;
    }
    randomPoints.updatePixels();
    fade = 1;
 
}
 
function draw() {
 
    clear();
    var positions = ctracker.getCurrentPosition();
 
    //debug
    // image(videoInput, 600, 0, 800, 600);
    // filter('GRAY');
 
 
    //mono image
    VideoGraphics.image(videoInput, -100, 0, 800, 600);
    VideoGraphics.filter('GRAY');
    Graphics.image(VideoGraphics, 0, 0, 800, 600);
    Graphics.filter('GRAY');
 
 
    //upper lip
    Graphics.stroke(180, 0, 0);
    Graphics.fill(120, 0, 0);
    Graphics.beginShape();
    for(var i=0; i&lt;=upperLip.length; i++){
        if(upperLip[i] &lt; positions.length){
            var index = upperLip[i];
            Graphics.curveVertex(positions[index][0], positions[index][1]);
            if(i == 0 || i == upperLip.length-1) Graphics.curveVertex(positions[index][0], positions[index][1]);
        }
    }
    Graphics.endShape();
 
    //lower lip
    Graphics.beginShape();
    for(var i=0; i&lt;=lowerLip.length; i++){
        if(lowerLip[i] &lt; positions.length){ var index = lowerLip[i]; Graphics.curveVertex(positions[index][0], positions[index][1]); if(i == 0 || i == lowerLip.length-1) Graphics.curveVertex(positions[index][0], positions[index][1]); } } Graphics.endShape(); FinalGraphics.push(); FinalGraphics.image(Graphics, -100, 0); FinalGraphics.pop(); //image(FinalGraphics,0,0); //draw shader shader.background(0); shader.fill(255); shader.shader(program); program.setUniform('resolution', [shader.width, shader.height]); program.setUniform('time', millis()); program.setUniform('image0', FinalGraphics); program.setUniform('image1', VideoGraphics); program.setUniform('randomPoints', randomPoints); program.setUniform('fade', fade); program.setUniform('noise', [noise(millis()), random()]); shader.rect(-shader.width/2 ,-shader.height/2, shader.width, shader.height); image(shader, 0, 0); drawFace(positions); //open the mouth or not if(positions.length &gt; 60){
        var l = positions[57][1] - positions[60][1];
        if(l &gt; 15){
            if(!openMouth){
                openMouth = true;
                startTime = Date.now();
            }
            var threshold = 1000;
            var fadeTime = 4000;
            var nt = Date.now() - startTime;
            if(nt &gt; threshold){ //if 1 seconds passed with opening mouth
                var f = (float)(nt - threshold)/fadeTime;
                fade = 1 - max(min(f, 1),0);
            }
        }
        else{
            openMouth = false;
            fade = 1;
        }
    }
}
 
 
function drawFace(positions){
 
    //view face
    noFill();
    var sc;
    if(fade &gt; 0.3) sc = 1;
    else if(fade &gt; 0.1) sc = (fade-0.1) * 5;
    else sc = 0;
    stroke(255, 255, 255, 255*sc);
    push();
    translate(-100, 0);
 
 
    //mouth
    beginShape();
    for(var i=0; i&lt;=upperLip.length; i++){
        if(upperLip[i] &lt; positions.length){
            var index = upperLip[i];
            curveVertex(positions[index][0], positions[index][1]);
            if(i == 0 || i == upperLip.length-1) curveVertex(positions[index][0], positions[index][1]);
        }
    }
    endShape();
    beginShape();
    for(var i=0; i&lt;=lowerLip.length; i++){
        if(lowerLip[i] &lt; positions.length){
            var index = lowerLip[i];
            curveVertex(positions[index][0], positions[index][1]);
            if(i == 0 || i == lowerLip.length-1) curveVertex(positions[index][0], positions[index][1]);
        }
    }
    endShape();
 
    /*
     //eye
     noFill();
     beginShape();
     for(var i=0; i&lt;=rightEye.length; i++){
     if(rightEye[i] &lt; positions.length){
     var index = rightEye[i];
     curveVertex(positions[index][0], positions[index][1]);
     if(i == 0 || i == rightEye.length-1) curveVertex(positions[index][0], positions[index][1]);
     }
     }
     endShape();
     beginShape();
     for(var i=0; i&lt;=leftEye.length; i++){
     if(leftEye[i] &lt; positions.length){
     var index = leftEye[i];
     curveVertex(positions[index][0], positions[index][1]);
     if(i == 0 || i == leftEye.length-1) curveVertex(positions[index][0], positions[index][1]);
     }
     }
     endShape();
 
     //nose
     beginShape();
     for(var i=0; i&lt;=nose.length; i++){
     if(nose[i] &lt; positions.length){
     var index = nose[i];
     curveVertex(positions[index][0], positions[index][1]);
     if(i == 0 || i == nose.length-1) curveVertex(positions[index][0], positions[index][1]);
     }
     }
     endShape();
     */
    pop();
}
var upperLip = [44, 45, 46, 47, 48, 49, 50, 59, 60, 61, 44];
var lowerLip = [44, 56, 57, 58, 50, 51, 52, 53, 54, 55, 44];
var nose = [33, 41, 62, 34, 35, 36, 42, 37, 43, 38, 39, 40];
var rightEye = [23, 63, 24, 64, 25, 65, 26, 66, 23];
var leftEye = [30, 68, 29, 67, 28, 70, 31, 69, 30];
 
 
 
var vert = `
#ifdef GL_ES
precision highp float;
precision highp int;
 
#endif
 
// attributes, in
attribute vec3 aPosition;
attribute vec3 aNormal;
attribute vec2 aTexCoord;
attribute vec4 aVertexColor;
 
// attributes, out
varying vec3 var_vertPos;
varying vec4 var_vertCol;
varying vec3 var_vertNormal;
varying vec2 var_vertTexCoord;
 
// matrices
uniform mat4 uModelViewMatrix;
uniform mat4 uProjectionMatrix;
//uniform mat3 uNormalMatrix;
 
void main() {
    gl_Position = uProjectionMatrix * uModelViewMatrix * vec4(aPosition, 1.0);
 
    // var_vertPos      = aPosition;
    // var_vertCol      = aVertexColor;
    // var_vertNormal   = aNormal;
    var_vertTexCoord = aTexCoord;
 
}`;
 
var frag = `
precision highp float;
#define PI 3.14159265359
varying vec2 var_vertTexCoord;
 
uniform vec2 resolution;
uniform float time;
uniform sampler2D image0;
uniform sampler2D image1;
uniform sampler2D randomPoints;
uniform float fade;
uniform vec2 noise;
 
void main(void)
{
 
    vec2 uv = var_vertTexCoord;
    float _noise = noise.x;
    float _random = 2.0 * noise.y - 1.0;
    vec2 st = 2.0*uv - 1.0;
 
    vec4 tex_color = texture2D(image0, uv);
 
    vec4 color = vec4(1.0);
 
    float dist = 1e10;
    float num = 30.0;
 
    for(int y=0; y&lt;30; y++){
        for(int x=0; x&lt;30; x++){
 
            vec2 index = vec2(float(x)/num, float(y)/num);
            vec4 c = texture2D(randomPoints, index);
            float i = float(x+int(num)*y)/(num*num);
 
            float newdist = distance(c.xy, uv);
            if(newdist &lt; dist){
                vec3 tc = texture2D(image0, c.xy).rgb;
                if (dist - newdist &lt; 0.01) {
                    float d = dist - newdist;
                    color.rgb = mix(vec3(0.0), tc, d/0.01);
                }else{
                    color.rgb = tc;
                }
                vec2 nc = c.xy * vec2(2.0) - vec2(1.0);
 
                float r = atan(nc.y, nc.x);
                r *= (5.0*cos(r*PI)+sin(r)) * PI;
                float len = length(nc)/sqrt(2.0);
                len += 0.05*sin(r);
                len = max(0.0, min(1.0,len));
                if( len &lt; fade){
                    // color.rgb = vec3(0.7);
                    // color.rgb = tex_color.rgb;
                    color.rgb = texture2D(image1, uv).rgb;
                }
                dist = newdist;
 
            }
        }
    }
    gl_FragColor = color;
 
}`;

paukparl-Body

 

Link (currently only works in Chrome):
paukparl.github.io/body

 

Sketch:

I figured I would track the nose and both the eyes to create 3 meta balls. Later, I added a fourth one to add some irregularity.

 

Code:

The code is really long and unorganized. I will clean it up later as I later make some improvements. The current code can be found here:
https://github.com/paukparl/sketches/blob/master/examples/test.html

 

Thoughts:

I wanted to make a real time 3D interactive sketch. While browsing through ThreeJS examples, I saw a sketch with moving isosurfaces, and I was instantly hooked. The way those balls formed surfaces based on proximity was really satisfying to watch.

I looked up how to make such surfaces. It could be done by something called "marching cubes." The underlying mechanisms of 3D graphics are really fascinating. It would have been really rewarding to make the whole sketch from scratch with WebGL, but I didn't have time. So I cloned the ThreeJS examples to my local repository and started making modifications to the original example code.

I originally wanted to work with refraction or reflection, but soon found out I would need to use something called cube maps. I couldn't figure out how to convert the flat webcam data to a cube map, so then I looked into texture mapping onto the isosurface. There is just one file dedicated to marching cubes in ThreeJS, and had some shortcomings, so I had to work around some things to get my current result.

Spoon-Body

https://youtu.be/H_0f3B_2JLY

This project ended up being more of an exploration of motion than an art piece. The goal was to take a motion capture skeleton and have it interact with some physics objects -- like springs and particles -- to produce an interesting piece. I achieved about half of that. Using the physics engine found in the Toxiclibs library in Processing, I attached a series of short springs and particles (which look like yellow strings in the video) to various points of the body. When the person moves, the springs move around and respond, accentuating the person's movement.

The affectation on the motion definitely came first. To be honest, the skeleton used in the piece was almost an afterthought. If I had left myself more time, I would have liked to get some motion capture footage of someone doing quick motions that cover a lot of space. This would have likely made the springs move all over the place and produce interesting paths and curves as the person moved.

 

 

import toxi.geom.*;
import toxi.geom.mesh.*;
import toxi.geom.mesh.subdiv.*;
import toxi.geom.mesh2d.*;
import toxi.geom.nurbs.*;
import toxi.physics3d.*;
import toxi.physics3d.behaviors.*;
import toxi.physics3d.constraints.*;
 
public class PBvh
{
  public BvhParser parser;  
 
  private ArrayList controlPoints;
  private ArrayList trailingPoints;
  private Particle finalPoint;
  private ArrayList springs;
  private VerletPhysics3D physics;
 
  private float w = 10;
 
  public PBvh(String[] data) {
    parser = new BvhParser();
    parser.init();
    parser.parse( data );
  }
 
  public void update( int ms ) {
    parser.moveMsTo( ms );//30-sec loop 
    parser.update();
  }
 
  public void setup() {
    controlPoints = new ArrayList();
    trailingPoints = new ArrayList();
    springs = new ArrayList();
    physics = new VerletPhysics3D();
    Vec3D gravity = new Vec3D(0, -10, 0);
    GravityBehavior3D gb = new GravityBehavior3D(gravity);
    physics.addBehavior(gb);
 
    finalPoint = new Particle(0, 0, 0);
 
    for( BvhBone b : parser.getBones())
    {
      if (true/*!b.hasChildren()*/)
      {
        Particle p1;
        ArrayList p2;
        p1 = new Particle(b.absEndPos.x, 
                          b.absEndPos.y, 
                          b.absEndPos.z);
        p2 = new ArrayList();
 
        for(int i = 0; i &lt; 10; i++) {
          Particle p = new Particle(b.absPos.x, 
                                    b.absPos.y, 
                                    b.absPos.z);
          p2.add(p);
        }
 
        p1.lock();
        ArrayList armSprings;
        armSprings = new ArrayList();//new VerletSpring3D(p1, p2.get(0), 5, 0.05);
        armSprings.add(new VerletSpring3D(p1,
                                          p2.get(0),
                                          1,
                                          0.1));
        for(int i = 1; i &lt; 10; i++) {
          VerletSpring3D s = new VerletSpring3D(p2.get(i - 1),
                                                p2.get(i),
                                                1,
                                                0.1);
          armSprings.add(s);
        }
        armSprings.add(new VerletSpring3D(armSprings.get(armSprings.size() - 1)
                                          finalPoint,
                                          1
                                          0.1)
        controlPoints.add(p1);
        physics.addParticle(p1);
        for(Particle p : p2) {
          trailingPoints.add(p);
          physics.addParticle(p);
        }
        for(VerletSpring3D spring : armSprings) {
          springs.add(spring); 
          physics.addSpring(spring);
        }
      } 
    }
  }
 
  public void draw() {
    fill(color(255));
    int counter = 0;
    for( BvhBone b : parser.getBones())
    {
      pushMatrix();
      translate(b.absPos.x, b.absPos.y, b.absPos.z);
      ellipse(0, 0, 2, 2);
      popMatrix();
 
      Particle p1 = controlPoints.get(counter);
      p1.x = b.absPos.x;
      p1.y = b.absPos.y;
      p1.z = b.absPos.z;
      counter++;
 
      if (!b.hasChildren())
      {
        pushMatrix();
        translate(b.absEndPos.x, b.absEndPos.y, b.absEndPos.z);
        println("xbody = ", b.absEndPos.x);
        println("ybody = ", b.absEndPos.y);
        println("zbody = ", b.absEndPos.z);
        ellipse(0, 0, 10, 10);
        popMatrix();  
 
        /*Particle p1 = controlPoints.get(counter);
        p1.x = b.absEndPos.x;
        p1.y = b.absEndPos.y;
        p1.z = b.absEndPos.z;
        counter++;*/
      }
 
    }
 
    physics.update();
    counter = 0;
    /*for(Particle p1 : controlPoints) {
      counter++;
      int bvhCounter = 0;
      for( BvhBone b : parser.getBones()) {
        if(!b.hasChildren()) {
          bvhCounter++;
        }
        if(bvhCounter == counter) {
          p1.lock();
          p1.x = b.absEndPos.x;
          p1.y = b.absEndPos.y;
          p1.z = b.absEndPos.z;
 
          println("xpoint = ", p1.x);
          println("ypoint = ", p1.y);
          println("zpoint = ", p1.z);
        }
      }
    }*/
 
    /*for(Particle p2 : trailingPoints) {
      p2.display();
    }*/
    for(VerletSpring3D spring : springs) {
      Particle p1 = (Particle) spring.a;
      Particle p2 = (Particle) spring.b;
 
      p1.display(2);
      p2.display(2);
 
      float x1 = p1.x;
      float y1 = p1.y;
      float z1 = p1.z;
 
      float x2 = p2.x;
      float y2 = p2.y;
      float z2 = p2.z;
 
      stroke(255, 198, 0);
      line(x1, y1, z1, x2, y2, z2);
    }
  }
}
// Originally from http://perfume-dev.github.io/
 
BvhParser parserA = new BvhParser();
PBvh /*bvh1,*/ /*bvh2,*/ bvh3;
 
public void setup()
{
  size( 1280, 720, P3D );
  background( 0 );
  noStroke();
  frameRate( 30 );
 
//  bvh1 = new PBvh( loadStrings( "A_test.bvh" ) );
//  bvh2 = new PBvh( loadStrings( "B_test.bvh" ) );
  bvh3 = new PBvh( loadStrings( "C_test.bvh" ) );
 
//  bvh1.update(0);
//  bvh2.update(0);
  bvh3.update(0);
 
//  bvh1.setup();
//  bvh2.setup();
  bvh3.setup();
 
  loop();
}
 
public void draw()
{
  background( 0 );
 
  //camera
  float _cos = cos(millis() / 7000.f);
  float _sin = sin(millis() / 5000.f);
  camera(width/8.f + width/8.f /** _sin +200*/, height/2.0f-100, 550 + 150/* * _cos*/, width/2.0f + 400, height/2.0f, -400, 0, 1, 0);
 
  //ground 
  /*fill( color( 255 ));
  stroke(127);
  line(width/2.0f, height/2.0f, -30, width/2.0f, height/2.0f, 30);
  stroke(127);
  line(width/2.0f-30, height/2.0f, 0, width/2.0f + 30, height/2.0f, 0);
  */stroke(255);
 
  pushMatrix();
  translate( width/2, height/2-10, 0);
  scale(-1, -1, -1);
 
  //model
//  bvh1.update( millis() );
//  bvh2.update( millis() );
  bvh3.update( millis() );
 
//  bvh1.draw();
//  bvh2.draw();
  bvh3.draw();
 
  popMatrix();
}
class Particle extends VerletParticle3D {
  Particle(float x, float y, float z) {
    super(x, y, z);
  }

void display(int n) {
pushMatrix();
translate(x, y, z);
//ellipse(0, 0, n, n);
popMatrix();
}
}


	


	





	

		
		
nerual-Body
	


	
	
	

		
It's clouded out there. Why go outside when you can vicariously experience the great outdoors as a cloud? Turn your head and blow yourself back and forth.


Video:




GIFs:






Image:




Process:


I really like clouds. One of the commonalities among some examples of face tracking I've seen is giving non-human objects human qualities, so I thought I'd try it with clouds. I was interested in a fun and cute interaction that lasted a few minutes.


These are some initial sketches and ideas for doing more than just blowing yourself around as a cloud. Unfortunately I didn't get to much beyond that.


 


I used this p5.js with Glitch and clmTracker template provided. I used Matter.js for the physics, although in rettrospect, the p5.js built in physics would have sufficed.


The CLM tracker gets an array of coordinates from the video input. Here is a screen shot I used to know which points went with which features, so I could isolate the eyes and mouth and calculate ratios to determine if the mouth was open or which way the face was facing.




It was very difficult to get the motion tracker to stabilize and reliably detect movements, so I didn't get around to implementing networking or my other ideas.


As I was transplanting the face points, I discovered some quirky movements that resulted, such as the interesting face created when the eye position, but not rotation, is tracked. It gave the cloud(I call him Claude) a wonky face that I liked. I originally left the mouth as a line/circle(like in the sketches), but I wanted to capture more of the user's orientation and movement, and let it dictate rather than just guide the movement. In general, I liked this aesthetic that arose from selectively misinterpreting and/or ignoring certain features.


Ultimately however, I feel like this project was a disappointment. Part of it was me not taking the time to set up the right structures to handle a bunch of bodies interacting with physics. But another part might have been deciding to go with an idea that relied mostly on a smooth final result, rather than having other aspects to lean on. I think I made some decent efforts with adding in the other clouds, but their movement(move way too fast when video input isn't showing), and in general the color scheme, are pretty lacking.


Code:



/* face-ctrl.js */
var ctracker;
//var mouth_pos;
var mouth_x = 300;
var mouth_y = 300;
var facingLeft, facingRight;
var blowRatio, turnRatio;
var m_width, m_height;
 
var ox,oy;
var prev;
var leye, reye;
var pos1 = [];
var pos2 = [];
var posMid = [];
 
function setup_clm(){
  //setup camera capture
  var videoInput = createCapture(VIDEO);
  videoInput.size(w/4,h/4); 
  //uncomment to view video
  //videoInput.position(w/4*3, h/4*3);
  //videoInput.hide();
 
  ctracker = new clm.tracker();
  ctracker.init(pModel);
  ctracker.start(videoInput.elt);
  mouth_x = 300;
  mouth_y = 300;
}
 
 
 
function updateVars(mx, my, scale){
  var positions = ctracker.getCurrentPosition();
 
  for (var i=positions.length-1; i&lt;positions.length; i++) { ox = positions[60][0]*scale; oy = positions[60][1]*scale; m_width = (positions[44][0] - positions[50][0]); m_height = (positions[47][1] - positions[53][1]); turnRatio = ((positions[44][0]-positions[1][0])/(positions[13][0]-positions[50][0])); facingLeft = turnRatio &gt; 1.5;
      facingRight = turnRatio &lt; 1;
  }
}
 
function isBlowing(){
  return blowRatio &lt; 2.5;
}
 
function drawBlowHole(mx, my, scale){
    var positions = ctracker.getCurrentPosition();
  blowRatio = m_width/m_height;
 
  push();
  noFill();
  strokeWeight(5);
  var mw = m_width*scale;
  var mh = m_height*scale;
 
  if (isBlowing()){
    //wind
    stroke(0);
    var bx1 = mx + (facingLeft ? -1 : 1)*mw*2;
    var bx2 = mx + (facingLeft ? -1 : 1)*mw*3;
    var by = my;
    var sep = 20;
    push();
    strokeWeight(3);
    line(bx1, by, bx2, by);
    line(bx1, by+sep, bx2, by+sep);
    line(bx1, by-sep, bx2, by-sep);
    pop();
  }
 
  //face
  for (var i=0; i&lt;positions.length; i++) {
    var x = positions[i][0]*scale;
    var y = positions[i][1]*scale;
    push();
    stroke(0);
    strokeWeight(scale*1.2);
    strokeJoin(ROUND);
    strokeCap(ROUND);
    //EYES
    if (i == 27 || i==32){
      stroke(0);
      noFill();
      pos1[0]= mx + x-ox;
      pos1[1] = my + y-oy +50;
      ellipse(pos1[0],pos1[1],40,20);
      fill(0);
      noStroke();
      ellipse(pos1[0],pos1[1],20,20);
    }
 
    //REAL MOUTH
    if(i&lt;44 || i&gt;61) continue;
    if (i==58 &amp;&amp; !isBlowing()){
      i=62;
      continue;
    }
    if (i == 55) pos2 = [positions[44][0]*scale,positions[44][1]*scale];
    else if (i == 61) pos2 = [positions[56][0]*scale,positions[56][1]*scale];
    else pos2 = [positions[i+1][0]*scale,positions[i+1][1]*scale,x,y];
 
    pos1[0]= mx + x-ox;
    pos1[1] = my + y-oy;
    pos2[0] = mx + pos2[0]-ox;
    pos2[1] = my +pos2[1]-oy;
    posMid[0] = (pos1[0] + pos2[0])/2;
    posMid[1] = (pos1[1] + pos2[1])/2;
 
    line(pos1[0],pos1[1],posMid[0],posMid[1]);
    line(pos2[0],pos2[1],posMid[0],posMid[1]);
  }
  pop();
}
 
/* sketch.js */
 
 
var socket = io();
var clients = {};
var data = {};
var w = 800;
var h = 600;
var size;
var videoInput;
var ctracker;
var claude;
var clouds = [{x:100,y:100,scalar:100, w:3, h:4},
              {x:200, y:h-100, scalar:-150, w:5, h:7},
              {x:w-300, y:h, scalar: 70, w:8, h:5},
              {x:w-100, y:200, scalar: -50, w:6, h:6}
             ]
var t = 0;
var Engine = Matter.Engine,
    Render = Matter.Render,
    World = Matter.World,
    Bodies = Matter.Bodies,
    Body = Matter.Body,
    Constraint = Matter.Constraint;
 
function setup_matterjs(){
  engine = Engine.create();
  world = engine.world;
  engine.world.gravity.y = 0;
  engine.timing.timeScale = 0.2;
  //makeBoundaries(width,height);
  Engine.run(engine);
}
 
function setup() {
  // setup canvas
  var cnv = createCanvas(w,h);
  cnv.position(0, 0);
  setup_clm(); //face-ctrl
  setup_matterjs();
  size = 50;
  claude = new Cloud(mouth_x, mouth_y, size);
  World.add(world, claude.body);
  started=0;
}
 
function drawCloudShape(x,y,h,w){  
  push();
  fill(250, 250, 250);
  noStroke();
  arc(x, y, 25 * h, 20 * h, PI + TWO_PI, TWO_PI);
  arc(x + 10*w, y, 25 * h, 45 * h, PI + TWO_PI, TWO_PI);
  arc(x + 25*w, y, 25 * h, 35 * h, PI + TWO_PI, TWO_PI);
  arc(x + 40*w, y, 30 * h, 20 * h, PI + TWO_PI, TWO_PI);
  pop();
}
 
function drawCloud(x,y,w,h){
  updateVars(x,y,w/2);
  drawCloudShape(x-w*10,y+h*7,w,h);
  drawBlowHole(x,y,w/2);
}
 
function drawSun(){
  push();
  noStroke();
  fill('#FFEC5E');
  var sun =  {x: map(hour(), 0, 60, 150, w-150), y: 150, r:200};
  ellipse(sun.x, sun.y, sun.r);
  pop();
}
 
function draw() {
  background('#A6D7FF');  
  drawSun();
  sass();
 
  push();
  //uncomment to mirror
  //translate(w,0);
  //scale(-1.0,1.0);
  strokeWeight(1);
  claude.update;
  drawCloud(claude.pos.x, claude.pos.y, 7, 5);
  pop();
 
  var wind = 0.1 * (facingLeft ? -1 : 1);
  var force = {x:wind,y:0};
  if (blowRatio &lt; 2) Body.applyForce(claude.body, claude.pos, force);
  for(var i =0; i&lt; clouds.length; i++) otherClouds(clouds[i]); } function sass(){ push(); if (claude.pos.x &gt; w+200 || claude.pos.x &lt; -200){
      textFont('Georgia');
      textSize(30);
      textAlign(CENTER);
      strokeWeight(1);
      fill(0);
      text("the real world doesn't have wrap-arounds, chump\n go outside",w/2,h/2);
    }
  pop();
}
 
function makeBoundaries(w,h){
  console.log("width="+str(w)+" height="+str(h));
  var strength = 20;
  var options = {isStatic:true, restitution: 0.2};
  var bottom = Bodies.rectangle(w/2, h, w+50, strength, options);
  var top = Bodies.rectangle(w/2, 0, w+50, strength, options);
  var left = Bodies.rectangle(0, h/2, strength, h+50, options);
  var right = Bodies.rectangle(w, h/2, strength, h+50, options);
  World.add(world,top);
  World.add(world,bottom);
  World.add(world,left);
  World.add(world,right);
}
 
function otherClouds(cloud){
  var scalar = cloud.scalar;
  var x1 = cloud.x + (scalar * cos(t));
  drawCloudShape(x1, cloud.y, cloud.w, cloud.h);
  //fill(0); noStroke();
  stroke(0); noFill();
  var dir = cloud.scalar &lt; 0 ? 1 : -1;
  var y = cloud.y-cloud.h*4;
  ellipse(x1 + cloud.w*15, y,20,10);
  ellipse(x1 + cloud.w*3, y, 20, 10);
  //ellipse(x1, cloud.y, 70, 70); //mouth?
  t+=0.005;
}


	


	





	

		
		
sapeck-Body
	


	
	
	

		
My goal was to create an environment of balloons where a motion-captured person interacts by popping the balloons. I chose a fight BVH from mocapdata.com that causes the figure to send a serious of punches at the wall of balloons. I looked through a lot of different fight sequences, and this one seemed to fit the best. I think it would have worked better if I had a continuously/endlessly walking and fighting person going through a never-ending stream of balloons. Otherwise, it's just boring.


I wanted to play with the placement of the balloons and having them move out of the way or drift off, but three.js made things difficult. For example, I found a good way to create a matte finish on the balloons, but that would prevent me from setting the opacity to hide the popped ones. I also found a good balloon 3D model, but I could not get three.js to display it. If I use three.js in the future, I need to have a much better understanding of it.



			

				
			

			
		









import colors from '../data/colorsHex'
 
var clock = new THREE.Clock()
 
var camera, controls, scene, renderer
var mixer, skeletonHelper
 
init()
animate()
 
var loader = new THREE.BVHLoader()
loader.load('bvh/fighting-31-syotei-yokoyama.bvh', result => {
  skeletonHelper = new THREE.SkeletonHelper(result.skeleton.bones[0])
  skeletonHelper.material.linewidth = 10
  skeletonHelper.skeleton = result.skeleton // allow animation mixer to bind to SkeletonHelper directly
 
  var boneContainer = new THREE.Group()
  boneContainer.add(result.skeleton.bones[0])
 
  scene.add(skeletonHelper)
  scene.add(boneContainer)
 
  // play animation
  mixer = new THREE.AnimationMixer(skeletonHelper)
  mixer.clipAction(result.clip).setEffectiveWeight(1.0).play()
})
 
// create an AudioListener and add it to the camera
var listener = new THREE.AudioListener()
camera.add(listener)
 
// create a global audio source
var sound = new THREE.Audio(listener)
 
// load a sound and set it as the Audio object's buffer
var audioLoader = new THREE.AudioLoader()
audioLoader.load('audio/Balloon Popping-SoundBible.com-1247261379.wav', buffer => {
  sound.setBuffer(buffer)
  sound.setLoop(false)
  sound.setVolume(1)
})
 
var ambientLight = new THREE.AmbientLight(0x000000)
scene.add(ambientLight)
 
var lights = []
lights[0] = new THREE.PointLight(0xffffff, 1, 0)
lights[1] = new THREE.PointLight(0xffffff, 1, 0)
lights[2] = new THREE.PointLight(0xffffff, 1, 0)
 
lights[0].position.set(0, 2000, 0)
lights[1].position.set(1000, 2000, 0)
lights[2].position.set(-1000, -2000, 0)
 
scene.add(lights[0])
scene.add(lights[1])
scene.add(lights[2])
 
let newBalloon = (r, color, x, y, z, o) => {
  var geometry = new THREE.SphereGeometry(r, 32, 32)
  var material = new THREE.MeshStandardMaterial({
    color: color,
    wireframe: false,
    transparent: true,
    opacity: o
  })
  var sphere = new THREE.Mesh(geometry, material)
  sphere.position.set(x, y, z)
  return sphere
}
 
let newBalloonGrid = (r, i, s, o) => {
  let balloons = []
  let pad = (r * 2) + s
  let c = ((i - 1) * pad) / 2
  for (let x of Array(i).keys()) {
    for (let y of Array(i - 4).keys()) {
      for (let z of Array(i - 2).keys()) {
        let color = colors[Math.floor(Math.random() * colors.length)]
        let bx = x * pad - c + 100
        let by = y * pad + r
        let bz = z * pad - c + 250
        let balloon = newBalloon(r, color, bx, by, bz, o)
        scene.add(balloon)
        balloons.push({
          pos: {
            x: bx,
            y: by,
            z: bz
          },
          r: r,
          o: o,
          color: color,
          mesh: balloon
        })
      }
    }
  }
  return balloons
}
let balloons = newBalloonGrid(20, 10, 5, 1)
 
function init () {
  camera = new THREE.PerspectiveCamera(90, window.innerWidth / window.innerHeight, 1, 1000)
  camera.position.set(0, 450, -400)
 
  controls = new THREE.OrbitControls(camera)
  controls.minDistance = 300
  controls.maxDistance = 700
 
  scene = new THREE.Scene()
 
  scene.add(new THREE.GridHelper(200, 10))
 
  // renderer
  renderer = new THREE.WebGLRenderer({ antialias: true })
  renderer.setClearColor(0xeeeeee)
  renderer.setPixelRatio(window.devicePixelRatio)
  renderer.setSize(window.innerWidth, window.innerHeight)
 
  document.body.appendChild(renderer.domElement)
 
  window.addEventListener('resize', onWindowResize, false)
}
 
function onWindowResize () {
  camera.aspect = window.innerWidth / window.innerHeight
  camera.updateProjectionMatrix()
 
  renderer.setSize(window.innerWidth, window.innerHeight)
}
 
var set = false
function animate () {
  // if (!isPlay) return
  window.requestAnimationFrame(animate)
 
  var delta = clock.getDelta()
 
  if (mixer) mixer.update(delta)
  // if (skeletonHelper) skeletonHelper.update()
 
  renderer.render(scene, camera)
 
  if (skeletonHelper) {
    if (!set) {
      console.log(skeletonHelper.skeleton.bones)
      set = true
    }
    if (skeletonHelper.skeleton) {
      for (let bone of skeletonHelper.skeleton.bones) {
        if (bone.name !== 'ENDSITE') {
          for (let balloon of balloons) {
            // console.log(skeletonHelper.skeleton.bones)
            let ballPos = balloon.pos
            let bonePos = bone.position
            let dist = Math.sqrt(Math.pow(ballPos.x - bonePos.x, 2) + Math.pow(ballPos.y - bonePos.y, 2) + Math.pow(ballPos.z - bonePos.z, 2))
            // console.log({ dist, ballPos, bonePos, name: bone.name })
            if (dist <= balloon.r * 4 && balloon.mesh.material.opacity !== 0) {
              console.log('KILL BALLOON')
              // console.log({ dist, ballPos, bonePos, name: bone.name })
              // scene.remove(balloon.mesh)
              if (balloon.mesh.material.opacity !== 0) {
                if (sound.isPlaying) sound.stop()
                sound.play()
              }
              balloon.mesh.material.opacity = 0
              // balloons.splice(balloons.indexOf(balloon))
              // scene.add(newBalloon(balloon.r, balloon.color, ballPos.x, ballPos.y, ballPos.z, balloon.o))
            }
          }
        }
      }
    }
  }
}


	


	





	

		
		
chewie-Body
	


	
	
	

		
Fullscreen: https://editor.p5js.org/full/B1LIV9a9m




I was interested in the idea of using corn on the cob as a visual medium. The kernels align in a way that is staggered uniformly. I also have never worked using pixel art with a hexagonal base, which ended up being surprisingly challenging.


I used a stock photo from this website. In photoshop I was able to create three different frames containing  "on" kernels spaced with enough room between them so that I could apply loose masks to display a single kernel at a time.






 


 


 


 


 


By using a rough mask I could isolate each kernel with enough room for variation so that the same mask could be used for each kernel while still getting precise edges.


Having never worked in hexagonal pixel space before, I opted to store the pixel data as 2d arrays because I figured it would be easier. It actually ended up being more difficult because I had to compensate for the offset columns of kernels. Especially with smaller, lower resolution sprites like this, even the difference of half of a pixel can make a big difference. I had to save two copies of each sprite so that when the 2d arrays are converted to the hexagonal pixels there isn't any inconsistency.


The following are a few sketches of the sprites. I thought about making the design rotate about a sort of sphere but I decided to keep the illusion 2-dimensional.


 


 



var kernels = [];
var activeKernels;
var colSize;
var rowSize;
var img;
var kern1;
var kern2;
var kern3;
var kernMask;
var randDelay;
 
var t = 0;
var blink = 0;
 
var faces = [
  [
    [
      [1, -1],
      [-1, -1],
      [-1, 1],
      [0, 1],
      [1, 1]
    ],
    [
      [1, -2],
      [-1, -2],
      [1, 0],
      [-1, 0],
      [0, 1]
    ]
  ],
  [
    [
      [2, -2],
      [-2, -2],
      [-2, 1],
      [-1, 2],
      [0, 2],
      [1, 2],
      [2, 1]
    ],
    [
      [2, -2],
      [-2, -2],
      [-2, 1],
      [-1, 1],
      [0, 2],
      [1, 1],
      [2, 1]
    ]
  ]
,[[[0,0],[0,-1]],[[0,0],[1,-1]]]];
 
var rat;
 
function preload() {
  img = loadImage("assets/corn_0.png");
  kern1 = loadImage("assets/corn_1.png");
  kern2 = loadImage("assets/corn_2.png");
  kern3 = loadImage("assets/corn_3.png");
  kernMask = loadImage("assets/corn_matte.png");
}
 
 
 
 
var ctracker;
 
function setup() {
  // setup camera capture
  randDelay = random(50,200);
  var videoInput = createCapture();
  videoInput.size(400, 300);
  videoInput.position(0, 0);
 
  // setup canvas
  cnv = createCanvas(800, 800);
  rat = 2000 / width;
  rowSize = height / 16;
  colSize = width / 11;
  loadKernels();
 
 
  cnv.position(0, 0);
  // setup tracker
  ctracker = new clm.tracker();
  ctracker.init(pModel);
  ctracker.start(videoInput.elt);
  noStroke();
}
 
function draw() {
  push();
 
  translate(width,0);
  scale(-1,1);
  clear();
 
  strokeWeight(5);
  image(img, 0, 0, width, height);
  //image(kern1,0,0,width, height);
  //image(kernels[ck[0]][ck[1]].img,0,0,width,height);
 
 
  // get array of face marker positions [x, y] format
  var positions = ctracker.getCurrentPosition();
  var p = positions[62];
  var q = positions[28];
  var r = positions[23];
 
 
 
//  for (var i = 0; i &lt; positions.length; i++) { // stroke("black"); //if (i == 23) stroke("green"); // set the color of the ellipse based on position on screen //fill(map(positions[i][0], width * 0.33, width * 0.66, 0, 255), map(positions[i][1], height * 0.33, height * 0.66, 0, 255), 255); // draw ellipse at each position point //ellipse(positions[i][0], positions[i][1], 8, 8); // } if (p != undefined) { var d = dist(q[0],q[1],r[0],r[1]); var size = 0; if (d&gt;110) size =1;
    if (d&lt;40) size =2; var ck = getCoors(map(p[0],0,255,0, 800),map(p[1],0,255,0, 800)); drawFace(ck[0], ck[1], size); } pop(); if (t&gt;=randDelay) {
    t=0;
    blink=7;
  	randDelay = random(50,200);
  }
  t++;
  if (blink&gt;0) blink--;
  print (p);
}
 
function Kernel(col, row) {
  this.col = col;
  this.row = row;
  this.x = colSize * (col + 0.5);
  this.y = rowSize * (row + 0.75);
  this.upCol = col % 2 == 0;
  if (this.upCol) this.y -= rowSize / 2;
 
  this.img = img;
  if (col % 2 == 0) {
    if (row % 3 == 0) this.img = kern2;
    if (row % 3 == 1) this.img = kern3;
    if (row % 3 == 2) this.img = kern1;
  } else {
    if (row % 3 == 0) this.img = kern1;
    if (row % 3 == 1) this.img = kern2;
    if (row % 3 == 2) this.img = kern3;
  }
 
  this.show = function() {
    var l = this.x - colSize;
    var gLeft = l * rat;
    var t = this.y - rowSize;
    var gTop = t * rat;
    var w = colSize * 2;
    var gWidth = w * rat;
    var h = rowSize * 2;
    var gHeight = h * rat;
    var dis = this.img.get(gLeft, gTop, gWidth, gHeight);
    dis.mask(kernMask);
    image(dis, l, t, w, h);
  };
}
 
function loadKernels() {
  var col = [];
  for (var c = 0; c &lt; 11; c++) {
    for (var r = 0; r &lt; 16; r++) {
      col.push(new Kernel(c, r));
    }
    kernels.push(col);
    col = [];
  }
}
 
function getCoors(x, y) {
  var col = x / colSize - 0.5;
  if (round(col) % 2 == 0) y += rowSize / 2;
  var row = y / rowSize - 0.75;
  if (col &lt; 0) col = 0; if (col &gt; 10) col = 10;
  if (row &gt; 15) row = 15;
  if (row &lt; 0) row = 0;
  return [int(round(col)), int(round(row))];
}
 
 
 
function drawFace(col, row, size) {
  var face = faces[size][0];
  if (col % 2 == 0) face = faces[size][1];
  var p;
  var c = 0;
  var r = 0;
  for (var i = 0; i &lt; face.length; i++) { p = face[i]; c = int(col + p[0]); r = int(row + p[1]); if ((c &gt;= 0 &amp;&amp; r &gt;= 0 &amp;&amp; c &lt; 11 &amp;&amp; r &lt; 16)&amp;&amp;
        !(blink!=0&amp;&amp;p[1]&lt;0))kernels[c][r].show();
  }
}


	


	





	

		
		
yuvian-Body
	


	
	
	

		

flappy face




note: this is not working 100% smoothly yet (collision detection is buggy sometimes)




For this project, I worked with clmtrackr.js in order to take input from the user's face to control a game. More specifically, I took note of where the user's face was positioned within the frame and the width of their smile. The more the user smiles, the higher the red "bird" flies. Moving his/her face from side to side also changes the bird's x-position on the canvas. I developed this project by getting clmtrackr to work first, then pulling data from the array of points it had given me, and then styling and implementing the game aspect. I didn't want to work from a motion capture because I wanted it to be more interactive, which is why I chose to get the user's face from his/her webcam. But with this also comes the potential for buggy face tracking as I have found while testing the project. I chose a flappy bird-esque game to implement because I wanted the game to be simple, easily understood, and somewhat recognizable. I had made another duplicate side-project in which the "bird" goes around eating berries but it seemed less like a game and more like a task.




how to play:

- smile widely to move bird up

- move head side to side to move bird left and right

- avoid white blocks



 smile 




 open mouth 






var ctracker;
var video;
var positions;
var bird;
var blockSpeed = 5
var blocks = [];
var score = 0;
var startGame = false;

function setup() {

  // setup webcam video capture
  video = createCapture(VIDEO);
  video.size(600, 450);
  video.position(0, 0);

  //hide video feed
  video.hide();

  // setup canvas
  createCanvas(600, 450);

  // setup tracker
  ctracker = new clm.tracker();
  ctracker.init(pModel);
  ctracker.start(video.elt);

  noStroke();

  // add block to blocks array
  for (var i = 0; i < 1; i++) {
    blocks[i] = new Block(random(40, 100), random(50, 100))
  }

}

function draw() {
  
  background(0);
  
  push();
  translate(video.width, 0);
  scale(-1, 1);

  image(video, 0, 0, width, height)

  positions = ctracker.getCurrentPosition();
  for (var i = 0; i < positions.length; i++) {
    fill(255, 80);
    // for when clmtracker is being weird in p5js editor
    // ellipse(positions[i][0] - 20, positions[i][1] - 10, 4, 4);
    // for online p5js editor
    ellipse(positions[i][0], positions[i][1], 2.5, 2.5);
  }
  pop();
  
  if (!startGame) {
    textAlign(CENTER)
    fill(255);
    text("press the space bar to start", width/2, height/2);
  }

  // if face is found, commence the game
  if (positions.length > 0 && startGame) {
    playGame()
  }
}

function keyPressed() {
  if (key == " ") {
    startGame = true
  } 
  else if (key == "R") {
    print('restart')
    setup();
  }
}


function playGame() {
    // blocks
    for (var i = 0; i < blocks.length; i++) {
      blocks[i].move();
      blocks[i].display();

      // remove block when it goes off screen and add a new block
      if (blocks[i].x < 25) {
        score += 1;
        blocks = []
        blocks.push(new Block(random(40, 100), random(50, 300)))
        // gradually increase speed of the moving blocks
        if (blockSpeed < 13) {
          blockSpeed = blockSpeed * 1.05
        } else {
          blockSpeed = blockSpeed * 1.005
        }
      }
  
      push();
        // display score
        fill(0);
        rectMode(CENTER);
        rect(80, 47, 70, 25)
        fill(255);
        textAlign(CENTER);
        text('score: ' + score, 80, 50);
      pop();
    }

    // background of smile bar
    fill(255);
    rect(20, 20, 15, height - 40);

    // smile bar
    var mouthLeft = createVector(positions[44][0], positions[44][1]);
    var mouthRight = createVector(positions[50][0], positions[50][1]);
    var smileDistance = Math.round(mouthLeft.dist(mouthRight));

    smile = map(smileDistance, 60, 75, 17, 0); // mapped smile distance to chopped up parts so bird doesn't jitter as much
    
    // prevent smile bar from going beyond bottom of bar
    rectSmile = smile * 20
    rectSmile = constrain(rectSmile, 20, height - 20)
    fill(50);
    rect(20, 20, 15, rectSmile);

    // smile number next to smile bar
    fill(255);
    smileText = height - 40 - Math.round(smile)
    // text(smileText, 50, constrain(smile, 30, height - 20))

    // bird
    push();
    translate(video.width, 0);
    scale(-1, 1);
    var noseX = positions[37][0]; // get nose x position to determine center of face
    noseX = noseX - 25 // wack p5js editor positioning
    birdY = smile * 20
    birdY = constrain(smile * 20, 50, height - 50); //prevent bird from moving offscreen
    bird = new Bird(noseX, birdY); // bird's x is nose position, y depends on width of smile
    bird.display();
    pop();
    
    // if bird collides with block, freeze everything and display game over message
    // for (var i = 0; i < blocks.length; i++) {
    block = blocks[0]
    if (bird.x >= block.x  && bird.x <= block.x + block.width) {
      bottomHeight = height - block.opening - block.height
      if (bird.y <= block.height || bird.y >= bottomHeight) {
        print('collision')
        // freeze webcam video, blocks, and bird
        video.stop()
        block.speed = 0
        // bird.y = height/2
        gameOver();
      }
    }
    
    // check top collision,
    // count = 0
    // d = getDistance(block.x, block.height, bird.x, bird.y)
    // if ( d < block.width/2) {
    //   count += 1
    //   print('hit' +  count)
    // }
  
    
    // alert user to fix positioning if bird is too high or too low
    if (smileText < 0) {
      fill(226, 74, 92);
      strokeWeight(3);
      text('move closer', 80, 100);
    }
    else if (smileText > 900) {
      fill(226, 74, 92);
      strokeWeight(3);
      text('move farther away', 80, 100);
    }
}

function gameOver() {
  fill(255);
  text("GAME OVER", width/2, height/2)
}

function Bird(x, y) {
  this.x = x;
  this.y = y;

  this.display = function() {
    fill(226, 74, 92);
    ellipse(this.x, this.y, 25, 25);
  }
}

function Block(w, h) {
  this.x = width - w / 2;
  this.width = w;
  this.height = h;
  this.speed = blockSpeed;
  this.opening = random(90, height / 2 - 50);

  this.display = function() {
    fill(255);
    // top block
    rect(this.x, 0, this.width, this.height)
      // bottom block
    rect(this.x, this.height + this.opening, this.width, height - this.opening - this.height)
  }

  this.move = function() {
    this.x = this.x - this.speed
  }
}

// from https://gist.github.com/timohausmann/5003280
function getDistance(x1,y1,x2,y2) {
  var 	xs = x2 - x1,
	ys = y2 - y1;		
	xs *= xs;
	ys *= ys;
	return Math.sqrt( xs + ys );
}


	


	





	

		
		
casher-body
	


	
	
	

		




(wow I look tired)






I had been intrigued by the last project's option to design a drawing program in Glitch, so I thought it would be fun to explore that in this project. I also wanted to create something that involves the whole body in order to make the user look ridiculous. Combining the two, I made this silly drawing application (which, I realized, can also double as an exercise "machine," if the user plays with it intensely enough). In my process, since this is a live application, I had to figure out how to track the different motions of various body parts and calculate their distances from specific points on the screen. Originally I had planned for the wideness of the opening of your mouth to control the stroke weight of the lines, but I discovered that using this code template, which doesn't analyze the mouth, was easier for me to use, so I had to change my idea a little bit. I am mostly happy with the product here -- it is very similar to the vision I had in the beginning, which is awesome because it means that I have either (or both) made progress in my coding abilities or learned to not be overly ambitious in my projects! If I could change a few things, I would first try to make it collaborative, so that people could draw together; and once I realized that it could be a form of exercise, I thought it could be cool to make it a game where the user would have to race a clock to draw something of specific shape, color, and size, which would force them to hop, squat, and move pretty quickly.


Directions:


You draw around the board with your nose. To turn on the pen, lift your left hand until it crosses the top white bar. Pick up the pen by touching the left bar with your left hand, and back up to the top to turn it on again. To change the color of the pen, jump high enough that both feet cross above the bottom white line. To make the pen thicker, jump so your nose crosses the top line; to thin the pen, duck low enough that your nose crosses the bottom line. To erase all lines, cross the right bar with your right hand.


 



let video;
let poseNet;
let poses = [];
let colors = ["black", "white", "red", "orange", "yellow", "green", "blue", "purple", "pink", "brown", "grey"]
let colorindex = 0;
let inArea = false;
let drawing = false;
let inArea2 = false;
let inArea3 = false;
let allLines = [];
let lines = [];
let circle = [65, 50, 30, 30];
let prevKpX;
let prevKpY;
let strokewidth = 3;
 
// Storing the last keypoint position
let lastKeypoints = [];
 
function setup() {
  createCanvas(640, 480);
  frameRate(24);
  video = createCapture({
    audio: false,
    video: {
      width: {
        ideal: 640
      },
      height: {
        ideal: 480
      }
    }
  });
  video.size(width, height);
 
  // Create a new poseNet method with a single detection
  poseNet = ml5.poseNet(video);
  // This sets up an event that fills the global variable "poses"
  // with an array every time new poses are detected
  poseNet.on('pose', function(results) {
    poses = results;
  });
  // Hide the video element, and just show the canvas
  video.hide();
 
  //print(lastKeypoints); 
  // setup original keypoints
  createDefaultKeypoints();
  //print(lastKeypoints);
}
 
// Create default keypoints for easing. 
function createDefaultKeypoints() {
  let numKeypoints = 17;
  for (let i = 0; i &lt; numKeypoints; i++) {
    newKeypoint = {
      x: width / 2,
      y: height / 2
    }
 
    lastKeypoints.push(newKeypoint);
  }
}
 
function updateKeypoints() {
  // If there are no poses, ignore it.
  if (poses.length &lt;= 0) {
    return;
  }
 
  // Otherwise, let's update the points; 
 
  let pose = poses[0].pose;
  let keypoints = pose.keypoints;
 
  for (let kp = 0; kp &lt; keypoints.length; kp++) {
 
    let oldKeypoint = lastKeypoints[kp];
    let newKeypoint = keypoints[kp].position;
 
    let interpX = lerp(oldKeypoint.x, newKeypoint.x, .3);
    let interpY = lerp(oldKeypoint.y, newKeypoint.y, .3);
 
    let interpolatedKeypoint = {
      x: interpX,
      y: interpY
    }
 
    lastKeypoints[kp] = interpolatedKeypoint;
  }
}
 
function draw() {
  push();
  translate(width, 0, 0);
  scale(-1, 1);
  image(video, 0, 0, width, height);
 
 
  updateKeypoints();
 
  drawKeypoints();
  pop();
 
  strokeWeight(5);
  stroke(255);
  fill(255);
  line(0, height * .9, width, height * .9);
  line(0, height *.1, width, height*.1);
  line(width*.1, 0, width*.1, height);
  line(width*.9, 0, width*.9, height);
  stroke(colors[colorindex]);
  ellipse(circle[0], circle[1], circle[2], circle[3]);
}
 
// A function to draw ellipses over the detected keypoints
function drawKeypoints() {
  for (let i = 0; i &lt; lastKeypoints.length; i++) {
    if (i == 0) {
      keypoint = lastKeypoints[i];
      fill(colors[colorindex]);
      //console.log(allLines);
      for (let i = 0; i &lt; allLines.length; i++) {
        for (let j = 0; j &lt; allLines[i].length; j++) { //console.log("c"); stroke(colors[allLines[i][j][4]]); strokeWeight(allLines[i][j][5]); line(allLines[i][j][0], allLines[i][j][1], allLines[i][j][2], allLines[i][j][3]); } } if (drawing == true) { if (lastKeypoints[9].y &gt; height * .1) {
          stroke(colors[colorindex]);
          //console.log(lines);
					lines[0] = [lastKeypoints[0].x, lastKeypoints[0].y, lastKeypoints[0].x, lastKeypoints[0].y, colorindex, strokewidth]
          x1 = prevKpX;
          y1 = prevKpY;
          x2 = lastKeypoints[0].x;
          y2 = lastKeypoints[0].y;
          lines.push([x1, y1, x2, y2, colorindex, strokewidth]);
          for (i = 1; i &lt; lines.length; i++) {
            stroke(colors[lines[i][4]]);
            strokeWeight(lines[i][5]);
            line(lines[i][0], lines[i][1], lines[i][2], lines[i][3]);
          }
          prevKpX = x2;
          prevKpY = y2;
        }
 
      }
      if (lastKeypoints[9].y &lt; height * .1) { if (drawing == false) { drawing = true; } } if (lastKeypoints[9].x &gt; width * .9) {
        if (drawing == true) {
          allLines.push(lines);
          lines = [];
          drawing = false;
        }
      }
      if (lastKeypoints[10].x &lt; width*.1) {
        allLines = [];
        lines = [];
        drawing = false;
      }
      if (lastKeypoints[0].y &lt; height*.1) {
        if (inArea2 == false) {
          if (circle[2] &lt; 80 &amp;&amp; circle[3] &lt; 80 &amp;&amp; strokewidth &lt; 52) { circle[2] += 5; circle[3] += 5; strokewidth += 3; inArea2 = true; } } } else { inArea2 = false; } if (lastKeypoints[0].y &gt; height*.9) {
        if (inArea3 == false) {
          if (circle[2] &gt; 10 &amp;&amp; circle[3] &gt; 10 &amp;&amp; strokewidth &gt; 3) {
            circle[2] -= 5;
            circle[3] -= 5;
            strokewidth -= 3;
            inArea3 = true;
          }
        }
      }
      else {
        inArea3 = false;
      }
      stroke(255);
      fill(255);
      strokeWeight(0);
      ellipse(lastKeypoints[0].x, lastKeypoints[0].y, 10, 10);
      ellipse(lastKeypoints[9].x, lastKeypoints[9].y, 20, 20);
      ellipse(lastKeypoints[10].x, lastKeypoints[10].y, 20, 20);
      ellipse(lastKeypoints[15].x, lastKeypoints[15].y, 20, 20);
      ellipse(lastKeypoints[16].x, lastKeypoints[16].y, 20, 20);
      if (lastKeypoints[15].y &lt; height * .9 &amp;&amp; lastKeypoints[16].y &lt; height * .9) {
        if (inArea == false) {
          colorindex = (colorindex + 1) % colors.length;
          inArea = true;
        }
      } else {
        inArea = false;
      }
    }
  }
}