Assignment: Synthesis: Music Instrument (group work)

Collaborators:

We turned the everyday interaction with a table and chair into a musical instrument.  By capturing the input of sitting in the chair and various leaning behaviors like leaning forward on the table, leaning on one elbow or leaning forward across the table, people can alter the music playing. 

We used 3 force sensors and a photocell to capture the interaction with the table and chair in the space.   The input is then fed into Processing to affect the pitch and volume of the music. 

We built a single person interaction prototype to demonstrate the concept.  It could easily be a collaborative instrument by building it into each side of the table, as well as multiple tables in a space.  

    •    Sitting down starts the music, and standing up turns it off
    •    Leaning forward over the table, engaging in a more intimate conversation decreases the volume to reflect the mood, whereas leaning back in a more social, open way increases the volume
    •    Leaning on one elbow, either in boredom or focus, lowers the pitch

Ideally Input from one person leaning on an elbow on the table, could mix with another person leaning back (or bouncing!) on the chair, forming a collaborative effect on the music in the room.  Add input from multiple people across multiple tables/chairs, and the music starts to reflect the energy in the room. 

This musical instrument is a small prototype of our final project idea, which is centered around capturing the implicit and unintentional behaviors of people within a space as input. We are exploring different outputs such as music, light, display, etc.  

Arduino Code

/* Musical Instrument
 * Arduino Code
 *
 * Andy Brooks
 * andy@ischool.berkeley.edu
 * November 12, 2008
 *
 */
 
// Analog Pin Settings

int FSRseatBack = 0; // FSR on the chair's seat back
int FSRseatPan = 1; // FSR on the seat/pan of the chair
int FSRtableTop = 2; // FSR on the tabletop; detects leaning pressure
int PHOTOtableTop = 3; // Photocell on the tabletop; detects leaning over tabletop

// Initialize values
int FSRseatBackVAL = 0;
int FSRseatPanVAL = 0; 
int FSRtableTopVAL = 0;
int PHOTOtableTopVAL = 0; 

// Primary method where we set up our serial communication
void setup() {
  Serial.begin(9600); // communicate at 9600 baud
}

// Method that continuously loops detecting values
void loop() {
  FSRseatBackVAL = analogRead(FSRseatBack);
  FSRseatPanVAL = analogRead(FSRseatPan); 
  FSRtableTopVAL = analogRead(FSRtableTop);
  PHOTOtableTopVAL = analogRead(PHOTOtableTop) * 4;
   
   
  Serial.print("a"); // seatback
  Serial.println(FSRseatBackVAL); // seatback value
  Serial.print("c"); // seatpan
  Serial.println(FSRseatPanVAL); // seatpan value
  Serial.print("e"); // FSR table 
  Serial.println(FSRtableTopVAL); // FSR table value
  Serial.print("g"); // photo table
  Serial.println(PHOTOtableTopVAL); // photo table value
 
  delay(50); // prevent the serial port from overload
}

Processing Code

// Nick Doty, Andy Brooks & Erin Knight
// code based on the sample:
//   Spooky Stream Save
//   by Krister Olsson <http://www.tree-axis.com>
//   Sound is generated in realtime, with pitch and 
//   pan controlled by the mouse.
// takes 4 inputs (pressure and photo sensors) to control the pitch and volume of the spooky audio streams

import krister.Ess.*;
import processing.serial.*;

String portname = "/dev/tty.usbserial-A7006xHW";
Serial port;
String buf=" ";
int cr = 13;  // ASCII return   == 13
int lf = 10;  // ASCII linefeed == 10

int oldCue = 0;

AudioStream myStream;
SineWave myWave1;
TriangleWave myWave2;

FadeOut myFadeOut;
FadeIn myFadeIn;
Reverb myReverb;

int frequency = 0;
float volume = 0.0;
float seatVolume = 0.0;
float deskVolume = 0.0;

int oldFrequency=0;
boolean doFadeIn=false;

boolean recording=false;
AudioFile myFile=new AudioFile();
int bytesWritten;

boolean on = false;

void setup() {
  port = new Serial(this, portname, 9600); 

  size(256,200);

  // start up Ess
  Ess.start(this);

  // create a new AudioStream
  myStream=new AudioStream();
  myStream.smoothPan=true;

  // our waves
  myWave1=new SineWave(0,.33);
  myWave2=new TriangleWave(0,.66);

  // our effects
  myFadeOut=new FadeOut();
  myFadeIn=new FadeIn();
  myReverb=new Reverb();

  // start
  myStream.start();
}

void draw() {
  int mx=mouseX;
  int my=height-mouseY;

  //various volume calculations based on photocell and chair back
  if (deskVolume < 0.2)
  {
   volume = deskVolume; 
  }
  else if (deskVolume > 0.2 && seatVolume < 0.2)
  {
    volume = deskVolume;
  }
  else if (deskVolume > 0.2 && seatVolume > 0.2)
  {
   volume = seatVolume; 
  }
  myStream.volume(volume);
  print("volume: "); println(volume);

  // clear the old
  noStroke();
  fill(0,0,255,64);
  rect(0,0,width,height);

  // paint new based on pan
  for (int i=0;i<width;i++) {
    stroke(255,abs(mx-i));
    line(i,0,i,height);
  }  

  // draw the curve
  stroke(255);

  // draw waveform
  int interp=(int)(((millis()-myStream.bufferStartTime)/(float)myStream.duration)*myStream.size);

  for (int i=0;i<256;i++) {
    float left=my;
    float right=my;

    if (i+interp+1<myStream.buffer2.length) {
      left-=myStream.buffer2[i+interp]*75.0;
      right-=myStream.buffer2[i+1+interp]*75.0;
    }

    line(i,left,i+1,right);
  }
}

void audioStreamWrite(AudioStream theStream) {
  if (!on)  //don't play until she sits down
  {
    myFadeOut.filter(myStream);
    //doFadeIn = false;
    println("I'm off!");
    return;
  }

  myWave1.generate(myStream);
  myWave2.generate(myStream,Ess.ADD);

  // adjust our phases
  myWave1.phase+=myStream.size;
  myWave1.phase%=myStream.sampleRate; 
  myWave2.phase=myWave1.phase;

  if (doFadeIn) {
    myFadeIn.filter(myStream);
    doFadeIn=false;
  }

  if (frequency!=oldFrequency) {
    // we have a new frequency    
    myWave1.frequency=frequency;

    // non integer frequencies can cause timing issues with our simple timing code
    myWave2.frequency=(int)(frequency*4.33); 

    myWave1.phase=myWave2.phase=0;

    // out with the old
    // fade out the old sound to create a 
    myFadeOut.filter(myStream);

    doFadeIn=true;
    println("Playing frequency: "+frequency);

    oldFrequency=frequency;
  } 

  // reverb
  myReverb.filter(myStream,.5); 
}

void serialEvent(Serial p) {
  int c = port.read();
  
  print((char)c);
  
  if (c != lf && c != cr) {
    buf += char(c);
  }
  else if (c == lf) {
    char inputType = buf.charAt(0);
    buf = buf.substring(1, buf.length());
    int val = int(buf);
    
    switch (inputType)
    {
     case 'a':  //seatback
       print("back: "); println(val);  //debug
       
       seatVolume = (val / 1000.0);
     break;
     case 'c':  //seatpan
       if (val < 50)
       {
          //on = false;   //optionally turn off when standing up
       }
       else
       {
         on = true;  //turn on when sitting down
       }
     break;
     case 'g': //photocell decreases the volume
        deskVolume = (val / 2000.0); 
          
        print("photo vol: "); println(deskVolume);  //debug
     break;
     case 'e': //table pressure
       //changes the pitch
       frequency = (val / 2) + 200;
     break;
    }

    buf = "";
  }
}

// we are done, clean up Ess
public void stop() {
  Ess.stop();
  super.stop();
}