User Interaction with my Project

For this stage, I had my brother test my project without explaining anything to him beforehand, and I recorded the interaction to see how he would naturally use it.

Overall, he understood most of it pretty quickly. He was able to figure out how to record audio, switch between modes, and use the buttons to trigger sounds. That made me feel like the core idea and interaction design are working well.

One of the main issues he noticed was with the sound samples, especially the piano and drums. They felt delayed when pressing the buttons, which made the system feel unresponsive. I realized this was because the audio files had a bit of silence at the beginning, so I went back and trimmed each sound manually so that they start exactly when the note or drum hit begins. That made a big difference in how responsive everything feels.

Another thing that confused him was the layout. The recordings were too close to the waveform visualization, which made the interface feel messy. Based on that, I moved all the recordings into a separate tab that you can open and close. That made the interface much cleaner and easier to understand.

He also tried to record the piano, drums, and bass directly, which wasn’t something I had implemented yet. At that point, the system only recorded audio from the microphone. This helped me realize that I needed to clearly separate those two types of recording. So I added two different options: one for recording voice through the mic, and another one for recording the full loop so it can repeat.

Another issue was that recordings kept accumulating and there was no way to delete them. He expected that to be possible, so I added a delete function inside the recordings panel.

In terms of understanding the system, everything else was pretty clear. The only thing he didn’t really understand was the potentiometer, mostly because he didn’t know what it was. The instructions explain it, but it’s still something that depends on the user’s familiarity with hardware.

Overall, his feedback was really helpful. It helped me fix small but important details like responsiveness, layout, and clarity.

At this point, the functionality is working really well. For the final version, I want to focus more on the design and make it feel more intentional and artistic, because right now it still looks pretty plain even though the interaction itself is solid.

 

Full p5js code till now

////// MUSIC PRODUCER FINAL //////

//INPUT FROM ARDUINO
let port;

let button1 = 0;
let button2 = 0;
let button3 = 0;
let button4 = 0;
let potentiometerValue = 0;

//STATE
let screen = "menu";
let mode = "piano";

//AUDIO SYSTEM
let mic, recorder;
let recordings = [];
let selectedRecordings = [];
let recordingNames = [];

// RECORDING STATES
let micRecording = null;
let loopRecording = null;
let isMicRecording = false;
let isLoopRecording = false;
let showRecordings = false;
let newRecordingAdded = false;

//SOUND LIBRARIES 
let pianoSounds = [];
let drumSounds = [];
let bassSounds = [];

//ACTIVE SOUND TRACKING (for button hold)
let activeSounds = {};

// UI ELEMENTS 
let connectButton, instructionsButton, startButton;
let bliseyFont;

// FFT VISUAL
let fft;

//LOAD ASSETS 
function preload() {

  bliseyFont = loadFont("Blisey.otf");

  pianoSounds[0] = loadSound("pianoC.mp3");
  pianoSounds[1] = loadSound("pianoD.mp3");
  pianoSounds[2] = loadSound("pianoE.mp3");
  pianoSounds[3] = loadSound("pianoF.mp3");

  drumSounds[0] = loadSound("kick.mp3");
  drumSounds[1] = loadSound("snare.mp3");
  drumSounds[2] = loadSound("hihat.mp3");
  drumSounds[3] = loadSound("clap.mp3");

  bassSounds[0] = loadSound("Bass1.mp3");
  bassSounds[1] = loadSound("Bass2.mp3");
  bassSounds[2] = loadSound("Bass3.mp3");
  bassSounds[3] = loadSound("Bass4.mp3");
}

//=== SETUP ===
function setup() {
  createCanvas(windowWidth, windowHeight);
  smooth();
  textFont(bliseyFont);

  port = createSerial();

  mic = new p5.AudioIn();
  mic.start();

  recorder = new p5.SoundRecorder();
  recorder.setInput(mic);

  fft = new p5.FFT(0.9, 128);

  setupMenu();
}

//=== MENU ===
function setupMenu() {
  removeAllButtons();

  instructionsButton = createButton("INSTRUCTIONS");
  instructionsButton.position(width/2 - 80, height/2);
  instructionsButton.mousePressed(goToInstructions);

  startButton = createButton("GET STARTED");
  startButton.position(width/2 - 80, height/2 + 50);
  startButton.mousePressed(goToMain);

  connectButton = createButton("CONNECT");
  connectButton.position(20, 20);
  connectButton.mousePressed(() => port.open(9600));
}

//== NAVIGATION ===
function goToInstructions() {
  removeAllButtons();
  screen = "instructions";

  let backButton = createButton("BACK");
  backButton.position(20, height - 60);
  backButton.mousePressed(setupMenu);
}

//=== MAIN INTERFACE ==
function goToMain() {
  removeAllButtons();
  screen = "main";

  // recordings tab toggle
  let recordingsTab = createButton("RECORDINGS");
  recordingsTab.position(20, 100);
  recordingsTab.mousePressed(() => {
    showRecordings = !showRecordings;
    newRecordingAdded = false;
  });

  // mic recording (voice only)
  let micButton = createButton("🎤");
  micButton.position(40, height - 180);
  micButton.size(70, 70);
  micButton.mousePressed(toggleMicRecording);

  // loop recording (full system)
  let loopRec = createButton("REC LOOP");
  loopRec.position(width/2 - 100, height - 100);
  loopRec.size(120, 50);
  loopRec.mousePressed(startLoopRecording);

  let loopStop = createButton("STOP LOOP");
  loopStop.position(width/2 + 40, height - 100);
  loopStop.size(120, 50);
  loopStop.mousePressed(stopLoopRecording);

  // mode selection
  let pianoButton = createButton("🎹");
  pianoButton.position(width - 100, 100);
  pianoButton.size(60, 60);
  pianoButton.mousePressed(() => mode = "piano");

  let drumsButton = createButton("🥁");
  drumsButton.position(width - 100, 180);
  drumsButton.size(60, 60);
  drumsButton.mousePressed(() => mode = "drums");

  let bassButton = createButton("🎸");
  bassButton.position(width - 100, 260);
  bassButton.size(60, 60);
  bassButton.mousePressed(() => mode = "bass");
}

//=== CLEAN BUTTONS ===
function removeAllButtons() {
  selectAll('button').forEach(b => b.remove());
}

//=== SERIAL READ ===
function readSerial() {
  let data = port.readUntil("\n");

  if (data) {
    let values = data.trim().split(",");

    if (values.length == 5) {
      button1 = int(values[0]);
      button2 = int(values[1]);
      button3 = int(values[2]);
      button4 = int(values[3]);
      potentiometerValue = int(values[4]);
    }
  }
}

//=== MIC RECORDING ===
function toggleMicRecording() {
  userStartAudio();

  if (!isMicRecording) {
    micRecording = new p5.SoundFile();
    recorder.record(micRecording);
    isMicRecording = true;
    port.println("LED:1");
  } else {
    recorder.stop();
    recordings.push(micRecording);
    recordingNames.push("Recording " + recordings.length);
    newRecordingAdded = true;
    isMicRecording = false;
    port.println("LED:0");
  }
}

//=== LOOP RECORDING ===
function startLoopRecording() {
  userStartAudio();

  loopRecording = new p5.SoundFile();
  recorder.record(loopRecording);

  isLoopRecording = true;
  port.println("LED:1");
}

function stopLoopRecording() {
  recorder.stop();

  recordings.push(loopRecording);
  recordingNames.push("Recording " + recordings.length);
  newRecordingAdded = true;

  loopRecording.loop(); // auto loop after recording

  isLoopRecording = false;
  port.println("LED:0");
}

//=== MAIN DRAW LOOP ===
function draw() {
  background(0);
  readSerial();

  if (screen === "menu") drawMenu();
  if (screen === "instructions") drawInstructions();
  if (screen === "main") drawMain();
}

//=== MENU DRAW ===
function drawMenu() {
  fill(255);
  textAlign(CENTER);
  textSize(40);
  text("MUSIC PRODUCER", width/2, height/3);
}

//=== INSTRUCTIONS DRAW ===
function drawInstructions() {
  fill(255);
  textAlign(CENTER);
  textSize(18);

  text(
`You are a DJ.

Record your voice and build music on top of it.

• Mic = voice recording
• Loop = full system recording

• Modes:
🎹 piano
🥁 drums
🎸 bass

• Buttons = trigger sounds
• Potentiometer = pitch control
• LED:
Red = idle
Green = recording`,
  width/2,
  height/2
  );
}

//=== MAIN DRAW ===
function drawMain() {
  noStroke();
  fill(255);
  textAlign(CENTER);
  textSize(30);
  text("MODE: " + mode, width/2, 60);

  drawWave();

  // recordings panel
  if (showRecordings) {
    
    noStroke();
    fill(20);
    rect(0, 150, 250, height);
    
    noStroke();
    fill(180);
    textAlign(CENTER);
    textSize(12);
    text("double click to rename a recording", 125, 170);

    textAlign(LEFT);

    for (let i = 0; i < recordings.length; i++) {
      let y = 200 + i * 40;

      fill(selectedRecordings.includes(i) ? "#00ff99" : 255);
      text(recordingNames[i], 20, y);
    }

    // delete button (panel only)
    let deleteX = 200;
    let deleteY = height - 60;
    
    noStroke();
    fill(120);
    rect(deleteX, deleteY, 30, 30, 5);

    noStroke();
    fill(255);
    textAlign(CENTER, CENTER);
    textSize(8);
    text("delete", deleteX + 15, deleteY + 15);
  }

  // new recording indicator
  if (newRecordingAdded) {
    
    fill(255, 200, 0);
    ellipse(140, 115, 10);
  }

  // pitch control for selected recordings
  let speed = map(potentiometerValue, 0, 1023, 0.5, 2);

  for (let i of selectedRecordings) {
    recordings[i].rate(speed);
  }

  handleButtons();
}

//=== AUDIO VISUALIZATION ===
function drawWave() {

  let waveform = fft.waveform();

  noFill();
  strokeWeight(3);

  stroke(255); drawLine(waveform, 0);
  stroke(0,255,200); drawLine(waveform, 40);
  stroke(255,100,100); drawLine(waveform, -40);
  stroke(100,100,255); drawLine(waveform, 80);
}

function drawLine(wave, offset) {

  beginShape();

  for (let i = 0; i < wave.length; i++) {
    let x = map(i, 0, wave.length, 0, width);
    let y = height/2 + wave[i] * 150 + offset;
    curveVertex(x, y);
  }

  endShape();
}

//=== HARDWARE BUTTONS ===
let p1=0,p2=0,p3=0,p4=0;

function handleButtons() {

  handleHold(1, button1);
  handleHold(2, button2);
  handleHold(3, button3);
  handleHold(4, button4);

  p1=button1; p2=button2; p3=button3; p4=button4;
}

//=== SOUND TRIGGER ===
function handleHold(n, state) {

  let sound;

  if (mode === "piano") sound = pianoSounds[n-1];
  if (mode === "drums") sound = drumSounds[n-1];
  if (mode === "bass") sound = bassSounds[n-1];

  if (state && !activeSounds[n]) {
    sound.play();
    activeSounds[n] = true;
  }

  if (!state && activeSounds[n]) {
    activeSounds[n] = false;
  }
}

//=== MOUSE INTERACTION ===
function mousePressed() {

  if (!showRecordings) return;

  for (let i = 0; i < recordings.length; i++) {

    let y = 200 + i * 40;

    if (mouseX < 250 && mouseY > y - 20 && mouseY < y + 20) {

      if (selectedRecordings.includes(i)) {
        recordings[i].stop();
        selectedRecordings = selectedRecordings.filter(r => r !== i);
      } else {
        recordings[i].loop();
        selectedRecordings.push(i);
      }
    }
  }

  // delete button click
  let deleteX = 200;
  let deleteY = height - 60;

  if (
    mouseX > deleteX &&
    mouseX < deleteX + 30 &&
    mouseY > deleteY &&
    mouseY < deleteY + 30
  ) {
    deleteSelected();
  }
}

//=== RENAME RECORDINGS ===
function doubleClicked() {

  if (!showRecordings) return;

  for (let i = 0; i < recordings.length; i++) {

    let y = 200 + i * 40;

    if (mouseX < 250 && mouseY > y - 20 && mouseY < y + 20) {

      let newName = prompt("Rename recording:");
      if (newName) {
        recordingNames[i] = newName;
      }
    }
  }
}

//=== DELETE RECORDINGS ===
function deleteSelected() {

  selectedRecordings.sort((a, b) => b - a);

  for (let index of selectedRecordings) {
    recordings[index].stop();
    recordings.splice(index, 1);
    recordingNames.splice(index, 1);
  }

  selectedRecordings = [];
}