User login
Assignment: Synthesis: Music Instrument (group work)
Collaborators:
Assignment: Synthesis: Music Instrument (group work)
Collaborators: bobacita, ash
Our toy takes input from a webcam, the computer keyboard, a potentiometer and a force sensor to create music from spinning gears. The webcam takes a picture of the gear setup and processes it to find where the gears are. A user can approve this by hitting a key to confirm that what the computer sees is correct. When the user spins the gears, the FSR records the vibration of the gear teeth hitting a popsicle stick and translates that to a rotational speed, and the potentiometer provides a value for volume.
What the webcam sees:
Our physical setup:
Our circuit:
The role of the Arduino is very simple: Every 10 times through the main loop, the number of 'clicks' from the FSR and the volume level are written to serial.
Arduino code:
/*
* FSR method for capturing gear rotation
* Monitor's FSR for input changes
* Outputs 0 or 1 depending on rate of change
*/
int forcePin = 0;
int potPin = 1;
int ledPin = 13;
int fOld = 0;
int f = 0;
int p = 0;
int frame = 0;
int count = 0;
void setup() {
Serial.begin(9600);
analogWrite(ledPin, 255);
}
void loop() {
f = analogRead(forcePin); // read the value from the sensor, 0-1023
p = analogRead(potPin); // read value from pot 0-1023
p = int(p/102.4);
//Serial.println(f);
if (f - fOld > 50) {
analogWrite(ledPin, 255);
count++;
} else {analogWrite(ledPin, 0);
}
if (frame % 10 == 0) {
Serial.print("p");Serial.println(p);
Serial.print("f");
Serial.println(count);
count = 0;
frame = 0;
}
fOld = f;
frame++;
delay(20);
}
Processing has the job of both identifying where the 'centers' (black pixels) of gears are and what colors those gears are. Pressing a key switches the program from analyse-mode to play-mode. In analyse-mode, Processing reads in pictures from a time-lapse program, finds where gears are, and prints them to a screen, looping every two seconds. In play mode, Processing remembers where the last known gears were, and plays associated sound loops when it gets a value for speed over serial.
Processing code:
import java.io.File;
import ddf.minim.*;
import processing.serial.*;
import processing.net.*;
Minim bMinim, gMinim, yMinim, oMinim;
AudioPlayer bSong, gSong, ySong, oSong;
AudioOutput bOut, gOut, yOut, oOut;
String portname = "COM7";
Serial port;
int gearMovement, volume;
//String orangeSong = "C:\\Documents and Settings\\criley\\My Documents\\Processing\\imageparser\\imageparser\\data\\ceramic.mp3";
String orangeSong = "ceramic.mp3";
String blueSong = "conga.mp3";
String greenSong = "maraca.mp3";
String yellowSong = "tabla.mp3";
boolean orangeGear = false;
boolean blueGear = false;
boolean greenGear = false;
boolean yellowGear = false;
String buf = "";
int cr = 13; // ASCII return == 13
int lf = 10; // ASCII linefeed == 10
PImage p;
int[][] poses = new int[100][2];
int gearcount = 0;
int thresh = 20;
int picCount = 10000;
int gearSize;
String picName;
boolean noclick = true;
int[][] blacks = new int[640][480];
void setup(){
size(640,480);
oMinim = new Minim(this);
bMinim = new Minim(this);
gMinim = new Minim(this);
yMinim = new Minim(this);
oOut = oMinim.getLineOut();
//oOut.printControls();
bOut = bMinim.getLineOut();
//bOut.printControls();
gOut = gMinim.getLineOut();
//gOut.printControls();
yOut = yMinim.getLineOut();
//yOut.printControls();
println(Serial.list());
//port = new Serial(this, Serial.list()[2], 9600);
port = new Serial(this, "COM7", 9600);
// this loads songs from the data folder
oSong = oMinim.loadFile(orangeSong);
bSong = bMinim.loadFile(blueSong);
ySong = yMinim.loadFile(yellowSong);
gSong = gMinim.loadFile(greenSong);
println("Setup ends.");}
void keyPressed(){
noclick=!noclick;
}
//Look for the next file from the time-lapse program; if you find one, draw it.
void draw(){ if(noclick){int pc = picCount+1;
File f = new File("C:\\Documents and Settings\\criley\\My Documents\\Processing\\imageparser\\imageparser\\data\\pic_"+pc+".jpg");
if(f.exists()){
picCount++;
picName = "pic_"+picCount+".jpg";
orangeGear = false;
blueGear = false;
greenGear = false;
yellowGear = false;
analyse();
//delay(2000);
drawEverything();
}
}else{drawEverything();
//println(orangeGear + yellowGear + blueGear + greenGear);
if(Integer.parseInt((str(gearMovement))) > 0) { if(orangeGear) {oSong.loop();
//println("Playing: " + orangeSong);}
if(yellowGear) {ySong.loop();
//println("Playing: " + yellowSong);}
if(greenGear) {gSong.loop();
// println("Playing: " + greenSong); }
if(blueGear) {bSong.loop();
// println("Playing: " + blueSong);}
delay(2000);
} else { if(orangeGear) {oSong.pause();
}
if(blueGear) {bSong.pause();
}
if(yellowGear) {ySong.pause();
}
if(greenGear) {gSong.pause();
}
}
if(oOut.hasControl(Controller.VOLUME)) {oOut.setVolume(volume*1023);
}
if(bOut.hasControl(Controller.VOLUME)) {bOut.setVolume(volume*1023);
}
if(yOut.hasControl(Controller.VOLUME)) {yOut.setVolume(volume*1023);
}
if(gOut.hasControl(Controller.VOLUME)) {gOut.setVolume(volume*1023);
}
}
}
void drawEverything(){//draw the image in the background
image(p,0,0);
noStroke();
//take 10 points as color samples for each gear
float r =0, g=0, b=0;
for(int i = 0; i<gearcount; i=i+1){ for(int j=0;j<10;j++){r=r+red(p.get(poses[i][0]-20,poses[i][1]+j));
g=g+green(p.get(poses[i][0]-20,poses[i][1]+j));
b=b+blue(p.get(poses[i][0]-20,poses[i][1]+j));
}
r=round(r/10);
g=round(g/10);
b=round(b/10);
//println(poses[i][0]+","+r+","+g+","+b);
color c = getColor((int)r,(int)g,(int)b);
fill(c);
//then draw the center circle and the color circle
if(gearSize==1){ellipse(poses[i][0], poses[i][1],125,125);} else if(gearSize==2){ellipse(poses[i][0], poses[i][1],185,185);} else if(gearSize==3){ellipse(poses[i][0], poses[i][1],230,230);}fill(0,0,0);
ellipse(poses[i][0], poses[i][1],10,10);
//println(gearcount);
}
}
color getColor(int r, int g, int b){
int mx = max(r,g,b);
float mult = 255/mx;
float rn = r*mult;
float gn = g*mult;
float bn = b*mult;
if(rn<200 && bn<210 && gn>128){//this is green
color c = color(0,255,0);
gearSize=2;
greenGear=true;
return c;
}else if(bn>gn && bn>rn){//this is blue
color c = color(0,0,255);
gearSize=3;
blueGear=true;
return c;
}else if(r>250){//this is orange
color c = color(255,128,0);
gearSize=1;
orangeGear=true;
return c;
}else if(abs(rn-gn)<60){//this is yellow
color c = color(255,255,0);
gearSize=3;
yellowGear=true;
return c;
}else{gearSize=1;
return color(rn,gn,bn);
}
}
void analyse(){
p = loadImage(picName);
p.loadPixels();
poses = new int[100][2];
gearcount = 0;
//Look at all the pixels
for(int i = 0; i<640; i=i+1){ for(int j=0; j<480; j=j+1){//if the pixel is reasonably black...
if(red(p.get(i,j))<=thresh && blue(p.get(i,j))<=thresh && green(p.get(i,j))<=thresh){blacks[i][j]=1;
if(gearcount>0){int diff=1;
//check if the pixel is really close to any others we've seen; if so they're probably the same center
for(int k=0; k<gearcount; k++){ if(abs((poses[k][0])-i) < 30 && abs((poses[k][1])-j) < 30){diff=0;
}
}
//if the pixel looks like it's unique to a new gear, enter it in the table
if(diff==1){//println(i+","+j);
gearcount++;
poses[gearcount-1][0]=i;
poses[gearcount-1][1]=j;
}
}else{gearcount++;
poses[gearcount-1][0]=i;
poses[gearcount-1][1]=j;
}
}else{blacks[i][j]=0;
}
}
}
}
void serialEvent(Serial port) {
int c = port.read();
if (c != lf && c != cr) {buf += char(c);
}
if (c == lf) {
char cmd = buf.charAt(0);
int val = int(buf.substring(1));
switch(cmd) {case 'p':
print("volume ");volume = val;
println(volume);
break;
case 'f':
print("gearMovement ");gearMovement = val;
println(gearMovement);
break;
}
buf = "";
}
}
void stop()
{ if(yellowGear){ySong.close();
}
if(blueGear){bSong.close();
}
if(greenGear){gSong.close();
}
if(orangeGear){oSong.close();
}
oMinim.stop();
bMinim.stop();
yMinim.stop();
gMinim.stop();
super.stop();
}