VR Archery

My game was all about creating a virtual archery game and linking a flex sensor to the bow, so it detects the bending motion and once bent the arrow shoots onto the target. For the theme of the entire game, I decided to link it with my childhood as well and use Minecraft as the main theme. As a 9 year old, I used to go to the archery range and play Minecraft at the time so it brought up a sense of nostalgic feeling.

For the bow, in order to give it a pixelated look, I decided to super glue the tiny wooden square blocks and paint over it. I attached the flex sensor on the top of the bow and attached a string to the flex sensor so when the string is pulled it shoots the arrow.

Future Improvements:

For future improvements, I’d like to add more visuals. I felt like it would’ve been more engaging if I did.

Pictures and Documentation:

Soldering documentation :

https://youtube.com/shorts/Cn9bonfPVeI?feature=share

Game play:

https://youtube.com/shorts/6olYsuVx5k8?feature=share

The Game & Codes:

<iframe src=”https://editor.p5js.org/sa6607/full/wcM5zq8Fr”></iframe>

P5 Code:

const serial = new p5.WebSerial();
let startButton;
let portButton;
let closeButton;
let sensorValue = 0;
let width = 900;
let height = 506;
let arrowSpeed = 5; // Speed at which arrow moves
let arrowDirection = 1;
let score = 0;
let shooting = false; // Indicates whether the arrow is currently being shot
let arrowX = width / 2; // X-coordinate of the arrow
let arrowY = height; // Y-coordinate of the arrow
let arrowScaleX = 1;
let arrowScaleY = 1;
let arrowWidth = 60;
let arrowHeight = 120;
let targetX = width / 2;
let targetY = height / 2;
let targetRadius = 100;

let started = false;

let bgImg;
let arrowImg;
let targetImg;
let sliderX = width - 50;
let sliderY = height / 2;
let lastScore = 0;
let textOpacity = 0;
let sliderHeight = 0;
let startShootFlag = 0;
let sliderIncrementor = 1;
let sliderTotalHeight = 100;
let startShootThreshHold = 40; //set flex sensor value at which we start for targetting

function allSerialStuff() {
  if (!navigator.serial) {
    alert("WebSerial is not supported in this browser. Try Chrome or MS Edge.");
  }
  // check for any ports that are available:
  serial.getPorts();
  // if there's no port chosen, choose one:
  serial.on("noport", makePortButton);
  // open whatever port is available:
  serial.on("portavailable", openPort);
  // handle serial errors:
  serial.on("requesterror", portError);
  // handle any incoming serial data:
  serial.on("data", serialEvent);
  serial.on("close", makePortButton);
  // add serial connect/disconnect listeners:
  navigator.serial.addEventListener("connect", portConnect);
  navigator.serial.addEventListener("disconnect", portDisconnect);
}

function serialEvent() {
  sensorValue = Number(serial.read());
  console.log(sensorValue);
  //if certain value from flex sensor get passed we get prepared for the shoot
  if (!shooting && sensorValue > startShootThreshHold) {
    startShootFlag = 1;
    sliderHeight = sensorValue + 20; // add 20 to elevate the value we need something in between 0-120
  }
  //if that value again crossed then we shoot
  if (!shooting && sensorValue < startShootThreshHold && startShootFlag) {
    startShootFlag = 0;
    if (!shooting && arrowY == height) {
      shooting = true; // Start shooting
    }
  }
}

// if there's no port selected,
// make a port select button appear:
function makePortButton() {
  // create and position a port chooser button:
  portButton = createButton("Choose Port");
  portButton.position(innerWidth / 2, 10);
  portButton.center("horizontal");
  // give the port button a mousepressed handler:
  portButton.mousePressed(choosePort);
}

// make the port selector window appear:
function choosePort() {
  if (portButton) portButton.show();
  serial.requestPort();
}

// open the selected port, and make the port
// button invisible:
// open the selected port, and make the port
// button invisible:
function openPort() {
  // wait for the serial.open promise to return,
  // then call the initiateSerial function
  serial.open().then(initiateSerial);

  // once the port opens, let the user know:
  function initiateSerial() {
    console.log("port open");
  }
  // hide the port button once a port is chosen:
  if (portButton) portButton.hide();
  makeCloseButton();
  if (closeButton) closeButton.show();
}

// pop up an alert if there's a port error:
function portError(err) {
  alert("Serial port error: " + err);
}
// read any incoming data as a string
// (assumes a newline at the end of it):

// try to connect if a new serial port
// gets added (i.e. plugged in via USB):
function portConnect() {
  console.log("port connected");
  serial.getPorts();
}

// if a port is disconnected:
function portDisconnect() {
  serial.close();
  console.log("port disconnected");
}

// if there's no port selected,
// make a port select button appear:
function makeCloseButton() {
  // create and position a port chooser button:
  closeButton = createButton("Close Port");
  closeButton.position(innerWidth / 2, 10);
  closeButton.center("horizontal");
  // give the close port button a mousepressed handler:
  closeButton.mousePressed(closePort);
}

function closePort() {
  serial.close();
  if (closeButton) closeButton.hide();
}

function preload() {
  bgImg = loadImage("/assets/background.jpg");
  targetImg = loadImage("/assets/target.png");
  arrowImg = loadImage("/assets/arrow.png");
}

function setup() {
  createCanvas(width, height);

  startButton = createButton("Start Game");
  startButton.addClass("start-button");
  startButton.position(innerWidth / 2, innerHeight / 2 + 10);
  startButton.center("horizontal");
  startButton.mousePressed(startGame);

  allSerialStuff();
}

function draw() {
  imageMode(CORNERS);
  image(bgImg, 0, 0, width, height);

  if (!started) {
    drawMenu();
  } else {
    // Draw target
    drawTarget();

    if (startShootFlag) {
      drawSlider();
    }

    //Draw score
    textAlign(LEFT, TOP);
    textSize(26);
    fill("red");

    text("Score: " + score, 10, 10);

    makeShooting();
    drawArrow();
    drawAddedScore();
  }
}

function drawMenu() {
  textSize(48);
  fill("#ff0033");
  textStyle(BOLD);
  textAlign(CENTER, BASELINE);
  text("VR Archery", width / 2, height / 2 - 100);
}

function startGame() {
  if (startButton) startButton.hide();
  started = true;
}

function makeShooting() {
  if (shooting) {
    // Calculate the trajectory towards the target
    let deltaY = height - targetY; // Difference in y-coordinates between arrow and target
    arrowY -= deltaY / 50; // Move the arrow towards the target
    if (arrowScaleY > 0.4) {
      arrowScaleY -= 0.004;
    }
    if (arrowScaleX > 0.4) {
      arrowScaleY -= 0.005;
    }
    // Stop shooting when arrow reaches the target
    if (arrowY - (arrowHeight * arrowScaleY) / 2 <= targetY) {
      shooting = false;

      // Check if the arrow hits the target
      let distance = dist(arrowX, arrowY - (arrowHeight * arrowScaleY) / 2, targetX, targetY); // Calculate distance between arrow tip and target center
      if (distance <= targetRadius - 20) {
        console.log("Hit!");
        arrowSpeed = 0;
        textOpacity = 255;
        updateScore(distance);
        //reset arrow after 2 seconds
        setTimeout(() => {
          arrowY = height;
          arrowScaleX = 1;
          arrowScaleY = 1;
          arrowSpeed = 5;
          textOpacity = 0;
        }, 2000);
      } else {
        console.log("Miss!");
        arrowY = height;
        arrowScaleX = 1;
        arrowScaleY = 1;
        shooting = false;
      }
    }
  } else {
    // Move arrow
    arrowX += arrowSpeed * arrowDirection;
    if (arrowX >= width || arrowX <= 0) {
      arrowDirection = -arrowDirection; // Reset arrow when it goes beyond the canvas
    }
  }
}

function drawTarget() {
  imageMode(CENTER);
  image(targetImg, targetX, targetY, targetRadius * 2, targetRadius * 2);
}

function drawArrow() {
  imageMode(CENTER);
  image(arrowImg, arrowX, arrowY, arrowWidth * arrowScaleX, arrowHeight * arrowScaleY);
}

function drawAddedScore() {
  fill(80, textOpacity);
  text("+ " + lastScore, targetX + 70, targetY - 60);
}

function drawSlider() {
  line(sliderX, sliderY, sliderX, sliderY + 120);
  line(sliderX - 10, sliderY, sliderX + 10, sliderY);
  line(sliderX - 10, sliderY + 60, sliderX + 10, sliderY + 60);
  line(sliderX - 10, sliderY + 120, sliderX + 10, sliderY + 120);
  fill(sliderHeight + 100, 200, 0);
  rect(sliderX - 10, sliderY, 20, sliderHeight);
}

//update score
function updateScore(distance) {
  console.log(distance);
  if (distance < 10) {
    score += 100;
    lastScore = 100;
  } else if (distance < 25) {
    score += 80;
    lastScore = 80;
  } else if (distance < 40) {
    score += 60;
    lastScore = 60;
  } else if (distance < 55) {
    score += 40;
    lastScore = 40;
  } else {
    score += 20;
    lastScore = 20;
  }
}

//handle mouse click event
function mouseClicked() {
  if (!shooting && arrowY == height) {
    //If arrow is not currently being shot
    shooting = true; // Start shooting
  }
}

function windowResized() {
  if (startButton) startButton.position(innerWidth / 2, innerHeight / 2 + 10).center("horizontal");
  if (portButton) portButton.center("horizontal");
  if (closeButton) closeButton.center("horizontal");
}

Arduino Code:

void setup() {
  Serial.begin(9600);
}

void loop() {
  int analogValue = analogRead(A0);
  byte byteToSend = map (analogValue, 0, 1023, 0, 255);
  Serial.write(byteToSend);
  delay(50);
}

The initial idea of this project is a bit different from what I ended up with. The initial idea was designed for two players, each placing one hand on a heart rate monitor. This game uses heart rate data to measure and display the level of affection or excitement between the participants. The faster the heartbeats, the higher the presumed love connection. But after I heard that we’d have to return the items we borrowed from the IM lab I thought it would be better if I order my own two heart rate monitors and solder them myself, then I wouldn’t have to take apart my hard work. With my not-so-good soldering skills I ruined one heart rate monitor and had only one to work with. I had to improvise and solve this issue working with only one heart rate monitor which led to changing the theme of the game a bit. Professor Aaron helped me with coming up with a new game that measures the players Happiness Level by reading their heart rates through the heart monitor. The game was initially supposed to start by pressing on a yellow button but due to time constrains and many other technical difficulties, such as linking the Arduino and the P5 together, I still managed to make it work fine with the keyboard even though I feel like the yellow button gives it more of a “gamey” feel to it which is what I would’ve much preferred.

Arduino Code:

const int buttonPin = 3;      // Pin where the button is connected
const int heartRatePin = A0;  // Analog pin for heart rate sensor

int heartRateValue = 0;
bool buttonPressed = false;

void setup() {
  pinMode(buttonPin, INPUT_PULLUP);  // Set the button pin as input with internal pull-up resistor
  pinMode(heartRatePin, INPUT);
  Serial.begin(9600);
  while (Serial.available() <= 0) {  // on/blink while waiting for serial data
    Serial.println("0,0");
    delay(50);
  }
}

void loop() {
  heartRateValue = analogRead(heartRatePin);  // Read the value from the heart rate sensor
  int bpm = calculateBPM(heartRateValue);     // Convert the analog reading to BPM

  // Read button state
  int buttonState = 1 - digitalRead(buttonPin);

 

  while (Serial.available()) {
  

    int left = Serial.parseInt();

    if (Serial.read() == '\n') {
      Serial.print(buttonState);
      Serial.print("0,50");
      Serial.println(bpm);
    }
  }


  // Always send the current heart rate and button state
}

// Function to simulate BPM calculation - replace this with your sensor-specific calculation
int calculateBPM(int sensorValue) {
  return sensorValue / 10;  // Simplified calculation for demonstration
}

P5.Js Code:

class MainMenu extends Menu {
  constructor(id) {
    super(id);

    this.pos = createVector(width / 2, height / 2.7);
    this.size = 240;
    this.strokeCol = color(
      random(100, 255),
      random(100, 255),
      random(100, 255)
    );
    this.hearts = [];
    for (let i = 0; i < 20; i++) {
      this.hearts.push({ x: random(width), y: random(height) });
    }
    this.random_seed = random(100, 10000); //use for heard animations in the back
    this.heartPos = { x: width / 2, y: height * 2 };
  }

  render() {
    background("#24182e");
    textAlign(CENTER, CENTER);
    textSize(44);
    textFont(pixel_font);
    fill("#8249c6");
    stroke(this.strokeCol);
    strokeWeight(4);

    text("HAPPINESS  LEVEL", width / 2, 50);
    //change strokcol every 20 farmes
    if (frameCount % 60 == 0) {
      this.strokeCol = color(
        random(100, 255),
        random(100, 255),
        random(100, 255)
      );
      this.random_seed = random(100, 10000);
    }
    textSize(30);
    stroke(200, 100, 100);
    push();
    randomSeed(this.random_seed);
    textFont("arial");
    for (let h of this.hearts) {
      for (let h2 of this.hearts) {
        if (dist(h.x, h.y, h2.x, h2.y) < 10) {
          strokeWeight(2);
          line(h.x, h.y, h2.x, h2.y);
        }
      }
      text("♥", h.x, h.y);
      h.x = lerp(h.x, random(width), 0.01);
      h.y = lerp(h.y, random(height), 0.01);
    }
    pop();
    push();
    textFont("arial");
    textSize(160);
    this.heartPos.y = lerp(this.heartPos.y, height / 1.5, 0.1);
    stroke(255);
    fill("#B28CDEAA");
    text("♥", this.heartPos.x, this.heartPos.y);
    textSize(30);
    noStroke();
    fill(255);
    textFont(pixel_font);
    text(
      "PLACE YOUR FINGER ON THE HEART",
      this.heartPos.x,
      this.heartPos.y + 100
    );
    pop();
    noStroke();
  }
}
class GameMenu extends Menu {
  constructor(id) {
    super(id);

    this.heart = new Heart(createVector(width / 2, height / 2.7), 240);
  }

  render() {
    textAlign(CENTER, CENTER);
    textSize(18); 
    background("#24182e");
    fill("#A88DC7");
    text("CHECKING YOUR LOVE LEVEL!", width / 2, height - 30);
    fill("#8249C67C");
    textFont("arial");
    textSize(34);

    for (let i = 0; i < 12; i++) {
      for (let j = 0; j < 8; j++) {
        let x = map(i, 0, 11, 0, width);
        let y = map(j, 0, 7, 0, height);
        if (frameCount % 40 < 20) {
          if (i % 2 == 0 || j % 2 == 0) {
            text("♥", x, y);
          }
        } else {
          if (i % 2 != 0 || j % 2 != 0) {
            text("♥", x, y);
          }
        }
      }
    }

    this.heart.render();
  }

  update() {
    this.heart.update();
    // Removed the timer decrement and check
  }

  reset() {
    this.heart = new Heart(createVector(width / 2, height / 2.7), 220);
  }
}
class EndMenu extends Menu {
  constructor(id) {
    super(id);

    this.finalScore = null;

    this.hearts = [];
    for (let i = 0; i < 2; i++) {
      this.hearts.push({ x: random(width), y: random(height) });
    }
    this.random_seed = random(100, 10000); //use for heard animations in the back
  }

  render() {
    background("#24182e");
    push();
    stroke(200, 100, 100);
    randomSeed(this.random_seed);
    textFont("arial");
    textSize(34);
    for (let h of this.hearts) {
      for (let h2 of this.hearts) {
        if (dist(h.x, h.y, h2.x, h2.y) < 100) {
          line(h.x, h.y, h2.x, h2.y);
        }
      }
      text("♥", h.x, h.y);
      h.x = lerp(h.x, random(width), 0.01);
      h.y = lerp(h.y, random(height), 0.01);
    }
    if (frameCount % 60 == 0) {
      this.random_seed = random(100, 10000);
    }
    pop();

    fill("#A88DC7");
    stroke(255);
    textFont(pixel_font);
    textSize(60);
    textAlign(CENTER, CENTER);
    text("THANK YOU !", width / 2, 160);
    noStroke();
    textSize(24);
    // text(
    //   `${this.finalScore}\n\nYOUR COMPATIBILITY SCORE`,
    //   width / 2,
    //   height / 1.5
    // );
    //     push();
    //     noStroke();
    //     fill(0);
    //     rect(0, 0, width, height);

    //     fill(255);
    //     textStyle(BOLD);
    //     textAlign(CENTER, CENTER);
    //     textSize(96);
    //     text("GAME OVER", width / 2, height / 4);

    //     textSize(40);
    //     text(`COMPATIBILITY SCORE: ${this.finalScore}`, width / 2, height / 2);

    //     textStyle(NORMAL);
    textSize(16);
    text("TRY AGAIN?", width / 2, height - 60);
    text("Yes", 100, height - 60);
    text("No", width - 100, height - 60);
    push();
    textFont("arial");
    pop();

    //     textSize(40);
    //     text("YES? OR NO?", width / 2, 640);

    //     pop();
  }

  reset() {
    this.finalScore = null;
  }
}

 I’m content with the final product despite getting extremely sick two days before the showcase while having other final projects due the day before it. For future improvements I’d like to incorporate my initial idea and also add a different game mode, Lie Detector Mode, which sounds fun to make with using the heart rate monitor. Overall, I feel like I got exposed to many things in this course which makes me much more comfortable with the things we’ve been working with during the entirety of this semester. I’d also like to thank professor Aaron for being extremely patient and helpful with me 🙂

The project’s concept:

The allure of a jukebox, with its nostalgic charm and tangible interaction with music selection, inspires a unique blend of past and present in your project. This modern reinterpretation of the classic jukebox isn’t just about listening to music—it’s an experiential dive into the ritual of choosing sounds from different eras and styles, echoing the tactile joy that came from flipping through vinyl records or pressing the physical buttons on a jukebox. Your project revives this delightful sensory interaction by blending physical buttons and digital outputs, allowing users to actively engage with the music rather than passively streaming playlists. It embodies a revival of the golden age of jukeboxes but with a contemporary twist, using today’s technology to recreate a piece of the past that resonates with both nostalgia and the new digital era.

The decision to incorporate a variety of music channels such as English, Classical, and Al Khalidiya channels suggests a celebration of diversity and the rich tapestry of global music culture. It reflects a yearning to bring the world closer together through the universal language of music, wrapped in the classic format of a jukebox. This project does more than just play music; it invites users to journey through different cultures and time periods at the push of a button. It’s a bridge between generations and geographies, enabling a shared experience that is both educational and entertaining, which is likely what sparked the idea to reinvent the jukebox for a modern audience. This blend of educational purpose and entertainment, rooted in technological innovation, makes your jukebox project a meaningful nod to the past while eagerly embracing the future of interactive media.

P5js code:

// Initial state settings
let loading = true;
let channels = [[], [], []]; // Arrays for storing songs by channel: 0 - English, 1 - Classical, 2 - Al Khalidiya
let numberOfSongs = 0;
let numberOfSongsLoaded = 0; 
let coinSound; // Sound effect for the coin insert
let selectedChannel; // Currently selected channel
let playing = false; // Is a song currently playing?
let songToPlay; // Current song playing

// Preload function to load sounds and other resources before the program starts
function preload() {
  soundPaths = loadStrings("soundFileNames.txt"); // Load list of sound file names
  coinSound = loadSound("sounds/coin.mp3"); // Load coin sound effect
}

// Setup function to initialize the environment
function setup() {
  createCanvas(600, 400);
  textAlign(CENTER, CENTER);

  // Loop through the sound paths and assign them to channels based on file names
  for (let i = 0; i < soundPaths.length; i++) {
    let words = soundPaths[i].split("_");
    let channel = words[words.length - 2] + "_" + words[words.length - 1]; // Determine the channel from the file name
    switch (channel) {
      case "english_channel.mp3":
        channels[0].push("sounds/" + words.join("_"));
        break;
      case "classical_channel.mp3":
        channels[1].push("sounds/" + words.join("_"));
        break;
      case "khalidiya_channel.mp3":
        channels[2].push("sounds/" + words.join("_"));
        break;
    }
  }
  numberOfSongs = soundPaths.length; // Total number of songs loaded
  // Load each song in the channels array
  for (let i = 0; i < channels.length; i++) {
    for (let j = 0; j < channels[i].length; j++) {
      channels[i][j] = loadSound(channels[i][j], () => {
        numberOfSongsLoaded += 1; // Increment the count of loaded songs
      });
    }
  }
  ratioPos = { x: width / 2, y: height * 2 };
  selectedChannel = floor(random(3)); // Randomly select a channel to start
  fft = new p5.FFT(); // Initialize Fast Fourier Transform for audio visualization
}

// Draw function to continuously execute and render the canvas
function draw() {
  background(40);
  if (loading) {
    // Show loading screen until all songs are loaded
    rectMode(CORNER);
    strokeWeight(2);
    textSize(34);
    fill(255);
    text("LOADING...", width / 2, height / 2 - 20);
    noStroke();
    fill(255);
    rect(width / 2 - 150, height / 2 + 20, 300, 40);
    fill(20);
    rect(
      width / 2 - 150,
      height / 2 + 20,
      map(numberOfSongsLoaded, 0, numberOfSongs, 0, 300),
      40
    );
    if (numberOfSongsLoaded == numberOfSongs) {
      loading = false;
    }
  } else {
    // Display the sound spectrum and UI once loading is complete
    let wave = fft.waveform();
    stroke(255, 50);
    noFill();
    beginShape();
    for (let i = 0; i < wave.length; i++) {
      let x = map(i, 0, wave.length, 0, width);
      let y = map(wave[i], -1, 1, height, 0);
      curveVertex(x, y);
    }
    endShape();

    rectMode(CENTER);
    ratioPos.y = lerp(ratioPos.y, height / 2, 0.1); // Smoothly move the UI element
    textSize(16);
    let channelName = "";
    switch (selectedChannel) { // Display the name of the selected channel
      case 0:
        channelName = " ENGLISH SONGS ";
        break;
      case 1:
        channelName = " CLASSICAL SONGS ";
        break;
      case 2:
        channelName = " KHALIDIYA SONGS ";
        break;
    }
    drawRadio(channelName, playing);
    drawChannels();
  }
}

// Event-driven functions to respond to keyboard presses for controlling the jukebox
function keyPressed() {
  switch (key) {
    case "n":
      nextChannel(); // Go to the next channel
      stopMusic(); // Stop the currently playing music
      break;
    case "b":
      prevChannel(); // Go to the previous channel
      stopMusic(); // Stop the music
      break;
    case " ":
      if (!playing) {
        playMusic(); // Start playing music if not already playing
      } else {
        stopMusic(); // Stop the music if playing
      }
      break;
  }
}

// Utility functions to control channels and playback
function nextChannel() {
  selectedChannel += 1; // Increment the channel index
  if (selectedChannel >= 3) {
    selectedChannel = 0; // Wrap around to the first channel
  }
}
function prevChannel() {
  selectedChannel -= 1; // Decrement the channel index
  if (selectedChannel < 0) {
    selectedChannel = 2; // Wrap around to the last channel
  }
}
function stopMusic() {
  if (songToPlay) {
    songToPlay.stop(); // Stop the currently playing song
  }
  playing = false;
}
function playMusic() {
  coinSound.play(); // Play the coin sound effect
  songToPlay = random(channels[selectedChannel]); // Select a random song from the current channel
  playing = true;
  songToPlay.loop(); // Start playing the selected song
}

// Drawing utility functions for UI elements
function drawChannels() {
  fill(100, 120, 100);
  rect(0, 150, 320, 70);
  fill(60, 70, 60);
  rect(0, 150, 300, 50);
  push();
  textAlign(LEFT, CENTER);
  let channels = ["English", "Classical", "Khalidiya"];
  textSize(12);
  for (let i = 0; i < 3; i++) {
    let x = 0;
    let y = 130 + 20 * i;
    noFill();
    if (selectedChannel == i) {
      fill(60, 90, 60);
      rect(x, y, 300, 15);
      fill(120, 150, 120);
      text("_" + channels[i], x - 150, y);
    } else {
      rect(x, y, 300, 15);
      fill(120, 150, 120);
      text(" " + channels[i], x - 150, y);
    }
  }
  pop();
}

// Function to draw the radio interface
function drawRadio(channel, playing = false) {
  translate(ratioPos.x, ratioPos.y);
  // Visual elements for the radio disk
  noStroke();
  fill(150, 150, 220, 100);
  circle(0, 100, 450);

  fill(20);
  circle(0, 100, 350);
  fill(200, 100, 100);
  circle(0, 100, 150);

  let channelName = channel.split("");
  push();
  translate(0, 100);
  if (playing) rotate(-frameCount / 60);
  push();
  for (let i = 0; i in channelName.length; i++) {
    rotate(TWO_PI / channelName.length);
    fill(255);
    text(channelName[i].toUpperCase(), 0, -50);
  }

  pop();
  pop();
  fill(180);
  circle(0, 100, 80);

  stroke(255);
  noFill();
  arc(0, 100, 420, 420, -PI / 2 + 0.4, -PI / 2 + 0.8);

  noStroke();
  strokeWeight(2);
  fill("#606A42");
  rect(0, 290, 500, 400, 40);
}

code with Arduino:

let loading = true;
let channels = [[], [], []]; //0 english channel , 1 - classical channel, 2 Al Khalidiya channel
let numberOfSongs = 0;
let numberOfSongsLoaded = 0;
let coinSound;
let selectedChannel;
let playing = false;
let songToPlay;
let toSend = 0;

function preload() {
  soundPaths = loadStrings("soundFileNames.txt");
  coinSound = loadSound("sounds/coin.mp3");

  // for (let i = 0; i < soundPaths.length; i++) {
  //   let words = soundPaths[i].split("_");
  //   //here we'll store sound paths in different arrays as different channels
  //   let channel = words[words.length - 2] + "_" + words[words.length - 1];
  //   switch (channel) {
  //     case "english_channel.mp3":
  //       channels[0].push("sounds/" + words.join("_"));
  //       break;
  //     case "classical_channel.mp3":
  //       channels[1].push("sounds/" + words.join("_"));
  //       break;
  //     case "khalidiya_channel.mp3":
  //       channels[2].push("sounds/" + words.join("_"));
  //       break;
  //   }
  // }
  // print(channels)
  // numberOfSongs = soundPaths.length;
  // //load every song in channel
  // for (let i = 0; i < channels.length; i++) {
  //   for (let j = 0; j < channels[i].length; j++) {
  //     channels[i][j] = loadSound(channels[i][j], () => {
  //       numberOfSongsLoaded += 1;
  //     });
  //   }
  // }
  /*
  ahlam_al_khalidiya_channel.mp3
cello_suite_classical_channel.mp3
gabl _aaarfak_al_khalidiya_channel.mp3
gymnopédie_classical_channel.mp3
i_lived_english_channel.mp3
mushfi_jorouhi_al_khalidiya_channel.mp3
overture_classical_channel.mp3
raheeb__al_khalidiya_channel.mp3
rouhe_thebak_al_khalidiya_channel.mp3
sundress_english_channel.mp3
swan_lake_suite_classical_channel.mp3
// sza_english_channel.mp3
virginia_beach_english_channel.mp3
whats_on_ur_mind_english_channel.mp3
  */

  {
    let sound = loadSound("sounds/i_lived_english_channel.mp3");
    channels[0].push(sound);
  }

  {
    let sound = loadSound("sounds/virginia_beach_english_channel.mp3");
    channels[0].push(sound);
  }
  
   
 


  
  
  
  
  
  
  
  
}

function setup() {
  createCanvas(600, 400);
  textAlign(CENTER, CENTER);

  //
  ratioPos = { x: width / 2, y: height * 2 };
  selectedChannel = floor(random(3));
  fft = new p5.FFT();
}

function draw() {
  background(40);
  if (loading) {
    rectMode(CORNER);
    strokeWeight(2);
    textSize(34);
    fill(255);
    text("LOADING...", width / 2, height / 2 - 20);
    noStroke();
    fill(255);
    rect(width / 2 - 150, height / 2 + 20, 300, 40);
    fill(20);
    rect(
      width / 2 - 150,
      height / 2 + 20,
      map(numberOfSongsLoaded, 0, numberOfSongs, 0, 300),
      40
    );
    if (numberOfSongsLoaded == numberOfSongs) {
      loading = false;
    }
  } else {
    //
    //draw Sound Spectrum
    let wave = fft.waveform();
    stroke(255, 50);
    noFill();
    beginShape();
    let x, y;
    for (let i = 0; i < wave.length; i++) {
      x = map(i, 0, wave.length, 0, width);
      y = map(wave[i], -1, 1, height, 0);
      curveVertex(x, y);
    }
    endShape();

    rectMode(CENTER);
    ratioPos.y = lerp(ratioPos.y, height / 2, 0.1);
    textSize(16);
    let channelName = "";
    switch (selectedChannel) {
      case 0:
        channelName = " ENGLISH SONGS ";
        break;
      case 1:
        channelName = " CLASSICAL SONGS ";
        break;
      case 2:
        channelName = " KHALIDIYA SONGS ";
        break;
    }
    drawRadio(channelName, playing);
    drawChannels();
  }
}

//functions to call from arduino's button
function keyPressed() {
  switch (key) {
    case "n":
      nextChannel();
      stopMusic();
      break;
    case "b":
      prevChannel();
      stopMusic();
      break;
    case " ":
      if (!playing) {
        playMusuc();
      } else {
        stopMusic();
      }
      break;
  }
}

function nextChannel() {
  selectedChannel += 1;
  if (selectedChannel >= 3) {
    selectedChannel = 0;
  }
}
function prevChannel() {
  selectedChannel -= 1;
  if (selectedChannel < 0) {
    selectedChannel = 2;
  }
}
function stopMusic() {
  if (songToPlay) {
    songToPlay.stop();
  }
  playing = false;
}
function playMusuc() {
  if (!playing) {
    coinSound.play();
    print("sel: " + selectedChannel);
    songToPlay = channels[selectedChannel][Math.floor(random(3))]; //random(channels[selectedChannel]);
    playing = true;
    songToPlay.loop();
  }
}

function drawChannels() {
  fill(100, 120, 100);
  rect(0, 150, 320, 70);
  fill(60, 70, 60);
  rect(0, 150, 300, 50);
  push();
  textAlign(LEFT, CENTER);
  let channels = ["English", "Classical", "Khalidiya"];
  textSize(12);
  for (let i = 0; i < 3; i++) {
    // text(channels[i],20,40+15*i);
    let x = 0;
    let y = 130 + 20 * i;
    noFill();
    if (selectedChannel == i) {
      fill(60, 90, 60);
      rect(x, y, 300, 15);
      fill(120, 150, 120);
      text("_" + channels[i], x - 150, y);
    } else {
      rect(x, y, 300, 15);
      fill(120, 150, 120);
      text(" " + channels[i], x - 150, y);
    }
  }
  pop();
}

function drawRadio(channel, playing = false) {
  translate(ratioPos.x, ratioPos.y);
  //disk
  noStroke();
  fill(150, 150, 220, 100);
  circle(0, 100, 450);

  fill(20);
  circle(0, 100, 350);
  fill(200, 100, 100);
  circle(0, 100, 150);

  let channelName = channel.split("");
  push();
  translate(0, 100);
  if (playing) rotate(-frameCount / 60);
  push();
  for (let i = 0; i < 20; i++) {
    rotate(TWO_PI / 20);
    fill(255);
    text("_", 0, -170);
  }
  pop();
  push();
  for (let i = 0; i < channelName.length; i++) {
    rotate(TWO_PI / channelName.length);
    fill(255);
    text(channelName[i].toUpperCase(), 0, -50);
  }

  pop();
  pop();
  fill(180);
  circle(0, 100, 80);

  stroke(255);
  noFill();
  arc(0, 100, 420, 420, -PI / 2 + 0.4, -PI / 2 + 0.8);

  noStroke();
  strokeWeight(2);
  fill("#606A42");
  rect(0, 290, 500, 400, 40);
}

function readSerial(data) {
  ////////////////////////////////////
  //READ FROM ARDUINO HERE
  ////////////////////////////////////

  if (data != null) {
    print(data);
    // make sure there is actually a message
    // split the message
    let fromArduino = split(trim(data), ",");
    // if the right length, then proceed
    if (fromArduino.length == 4) {
      // only store values here
      // do everything with those values in the main draw loop

      // We take the string we get from Arduino and explicitly
      // convert it to a number by using int()
      // e.g. "103" becomes 103
      let start = int(fromArduino[0]);
      let stop = int(fromArduino[1]);
      let next = int(fromArduino[2]);
      let prev = int(fromArduino[3]);

      if (start == 1) {
        print("start");
        playMusuc();
      } else if (stop == 1) {
        print("stop");
        stopMusic();
      } else if (next == 1) {
        print("next");
        nextChannel();
        stopMusic();
      } else if (prev == 1) {
        print("prev");
        prevChannel();
        stopMusic();
      }

      //////////////////////////////////
      //SEND TO ARDUINO HERE (handshake)
      //////////////////////////////////
      let sendToArduino = 0 + "\n";
      writeSerial(sendToArduino);
    }
  }
}

function keyPressed() {
  if (key == " ") {
    // important to have in order to start the serial connection!!
    setUpSerial();
  }
}

Showcase:

Struggles:

When I initially incorporated Arduino buttons into the program, I envisioned a seamless integration that would enhance the user experience by providing tangible controls for the digital jukebox. However, this integration proved to be more challenging than anticipated. The entire program malfunctioned, leading to unexpected behaviors where buttons would either not respond or trigger incorrect actions. This technical hurdle was a significant setback, as it compromised the core functionality of the project—interacting with the music playlist through physical controls.

Faced with these difficulties, I realized that a pivot was necessary to maintain the integrity and usability of the project. I revisited the drawing board, reevaluating the interfacing between the Arduino and the software. This required stripping down complex parts of the code, simplifying the communication protocols, and implementing more robust error handling mechanisms to ensure that each button press accurately corresponded to the intended action. Although this pivot was a detour from my original vision, it was a crucial learning experience that emphasized the importance of adaptability and thorough testing in the development process. The revamped approach not only resolved the issues but also reinforced the project’s functionality, making it more reliable and user-friendly.

Future Improvements:

Based on the experiences and challenges encountered with the integration of Arduino buttons and the development of the digital jukebox, several future improvements can be outlined to enhance the project further:

1. Enhanced Error Handling and Debugging Tools: Implementing more sophisticated error handling mechanisms can help identify and resolve issues more efficiently when they arise during the interaction between the Arduino hardware and the software. Additionally, developing a suite of debugging tools or visual indicators in the software can help monitor the state and health of the system in real-time.

2. User Interface Improvements: Enhancing the user interface to provide clearer feedback and more intuitive controls can significantly improve the user experience. This could include visual indicators of button presses, more responsive animations, or a more aesthetically pleasing layout that mimics the classic jukebox style.

3. Expanded Music Library and Categorization: Expanding the music library to include more diverse genres and new channels can cater to a broader audience. Implementing a more dynamic categorization system where users can create custom playlists or choose from themed channels could add a new layer of interaction.

4. Wireless Control Options: Introducing wireless control options such as Bluetooth or Wi-Fi connectivity could allow users to control the jukebox from their smartphones or other devices. This could be particularly useful for accessibility purposes and to accommodate larger venues.

5. Improved Audio Quality and Effects: Upgrading the sound output hardware or integrating software that allows for better sound equalization and effects can enhance the overall listening experience. This might include features like bass boost, echo, and balance adjustments that users can control directly.

6. Sustainability and Maintenance: Considering the long-term sustainability and ease of maintenance in the design can ensure the jukebox remains functional and enjoyable for years. This could involve using more durable materials for the hardware, making the system modular for easy repairs, or providing software updates to keep the system secure and efficient.

7. Interactive Features and Gamification: Introducing interactive features such as music trivia, user contests, or gamification elements where users can earn points or rewards for their interaction can increase engagement and provide a more entertaining experience.

These improvements aim to refine the functionality, broaden the appeal, and ensure the longevity of the digital jukebox project, making it not only a nostalgic piece but a cutting-edge feature for any social or personal space.

Arduino code:

// Define the pin numbers for the buttons
const int pinGreen = 11; // Start
const int pinYellow = 9; // Stop
const int pinBlue = 5; // Next Channel
const int pinBlack = 3; // Previous Channel

void setup() {
  // Initialize the buttons as inputs
  pinMode(pinGreen, INPUT);
  pinMode(pinYellow, INPUT);
  pinMode(pinBlue, INPUT);
  pinMode(pinBlack, INPUT);

  // Start serial communication at 9600 bps
  Serial.begin(9600);
  // start the handshake
  while (Serial.available() <= 0) {
    Serial.println("0,0,0,0"); // send a starting message
    delay(50);
  }
}

void loop() {
  // Read the state of each button
  int stateGreen = digitalRead(pinGreen);
  int stateYellow = digitalRead(pinYellow);
  int stateBlue = digitalRead(pinBlue);
  int stateBlack = digitalRead(pinBlack);
  // Serial.println(toSend);

  // String toSend;

  // Send different commands based on button presses
  // if (stateGreen == HIGH) {
  //   toSend = "start";
  // }
  // else if (stateYellow == HIGH) {
  //   Serial.println("stop"); // Command to stop the radio
  // }
  // else if (stateBlue == HIGH) {
  //   Serial.println("next"); // Command to go to next channel
  // }
  // else if (stateBlack == HIGH) {
  //   Serial.println("prev"); // Command to go to previous channel
  // }

  while (Serial.available()) {
    int fromP5 = Serial.parseInt();
   
    if (Serial.read() == '\n') {
      Serial.print(stateGreen);
      Serial.print(",");
      Serial.print(stateYellow);
      Serial.print(",");
      Serial.print(stateBlue);
      Serial.print(",");
      Serial.println(stateBlack);
    }
  }

  // delay(100); // Delay to debounce and prevent multiple sends
}

Concept

For my final project, I decided to create a Photo Booth that questions and criticises the act of taking an individual selfie while encouraging the processes of documenting memories with others. The purpose of the work is to discourage and eliminate the ego-centric approach behind how we make use of our smartphone’s camera. In the same way that the widespread use of smartphones has increased the awareness of responsible online practices, I wanted to create something that spoke to the importance of how we utilise this incredibly accessible method of documenting our lives. Essentially, Selfie4Two is a work that focuses on celebrating interpersonal connection through technology.

Technical Approach & Challenges

Initially, my approach in creating this Photo Booth concept was to utilise machine learning technology through an image classification model. This model would discern whether there were one or two people in the frame and whether they were holding up their phone (suggesting that they are taking a picture of themselves). I would then incorporate this model into p5js and load an ASCII webcam rendering based on what was detected, presenting question marks for one person and hearts for groups. Screenshots of the image classification model are seen below.

I was able to functionally incorporate it into p5 with some initial testing but could not effectively do so with the ASCII webcam rendering that I had made, meaning I was forced to avoid using it in order to finish the project.  This certainly limited the usability of the project as well as minimising how effective the concept was conveyed. In an attempt to resolve this, I included emojis (maintaining the theme of internet culture) onto the initial webcam display that can be clicked to display the ASCII renderings.

In terms of the ASCII webcam rendering, I was able to create two variations with one being comprised of question marks and the other of hearts. These are both displayed below.

After incorporating these into my final code, however, they ran much slower and did not display correctly. Unfortunately, I could not figure out how to fix this and had to present this version of the code as my final project.

const density = "  \u2661";
const density1 = "  ?";
let video;
let emojiSize = 32; // Size of the emojis

function setup() {
  createCanvas(window.innerWidth, window.innerHeight);
  video = createCapture(VIDEO);
  video.hide();
  let button = createButton("reset");
  button.mousePressed(resetSketch);
}

function draw() {
  background(0);
  video.loadPixels();

  // Draw the video
  image(video, 0, 0, width, height);

  let emoji = "🤳🧍❔";
  let emoji1 = "🤳🧑‍🤝‍🧑❤️";

  text(emoji1, width / 6, height / 4);
  text(emoji, width / 1.2, height / 4);
  textSize(32);
  textAlign(CENTER, CENTER);

  // Calculate bounding boxes for emojis
  let emojiBox = {
    x: width / 6 - emojiSize / 2,
    y: height / 4 - emojiSize / 2,
    width: textWidth(emoji),
    height: emojiSize,
  };

  let emojiBox1 = {
    x: width / 1.2 - emojiSize / 2,
    y: height / 4 - emojiSize / 2,
    width: textWidth(emoji1),
    height: emojiSize,
  };

  // Check if mouse click is inside the bounding box of the first emoji
  if (
    mouseX > emojiBox.x &&
    mouseX < emojiBox.x + emojiBox.width &&
    mouseY > emojiBox.y &&
    mouseY < emojiBox.y + emojiBox.height
  ) {
    coupleAscii();
  }

  // Check if mouse click is inside the bounding box of the second emoji
  if (
    mouseX > emojiBox1.x &&
    mouseX < emojiBox1.x + emojiBox1.width &&
    mouseY > emojiBox1.y &&
    mouseY < emojiBox1.y + emojiBox1.height
  ) {
    singleAscii();
  }
}

function singleAscii() {
  clear();
  textSize(10);
  background(0);
  video.loadPixels();
  fill(255, 180, 180);
  stroke(255, 180, 180);
  strokeWeight(1);
  let asciiImage1 = "";
  for (let j = 0; j < video.height; j++) {
    for (let i = 0; i < video.width; i++) {
      const pixelIndex = (i + j * video.width) * 4;
      const r = video.pixels[pixelIndex + 0];
      const g = video.pixels[pixelIndex + 1];
      const b = video.pixels[pixelIndex + 2];
      const avg = (r + g + b) / 3;
      const len = density1.length;
      const charIndex = floor(map(avg, 0, 255, 0, len));
      const c = density1.charAt(charIndex);
      text(
        c,
        map(i, 0, video.width, 0, width),
        map(j, 0, video.height, 0, height)
      );
    }
  }
}

function coupleAscii() {
  clear();
  textSize(10);
  background(0);
  video.loadPixels();
  fill(255, 180, 180);
  stroke(255, 180, 180);
  strokeWeight(1);
  let asciiImage = "";
  for (let j = 0; j < video.height; j++) {
    for (let i = 0; i < video.width; i++) {
      const pixelIndex = (i + j * video.width) * 4;
      const r = video.pixels[pixelIndex + 0];
      const g = video.pixels[pixelIndex + 1];
      const b = video.pixels[pixelIndex + 2];
      const avg = (r + g + b) / 3;
      const len = density.length;
      const charIndex = floor(map(avg, 0, 255, 0, len));
      const c = density.charAt(charIndex);
      text(
        c,
        map(i, 0, video.width, 0, width),
        map(j, 0, video.height, 0, height)
      );
    }
  }
}
function resetSketch() {
  clear();
  background(0);
  video.loadPixels();

  // Draw the video
  image(video, 0, 0);

  push();
  let emoji = "🤳🧍❔";
  let emoji1 = "🤳🧑‍🤝‍🧑❤️";

  text(emoji1, width / 6, height / 4);
  text(emoji, width / 6, height / 2);
  textSize(32);
  textAlign(CENTER, CENTER);
  pop();

  // Calculate bounding boxes for emojis
  let emojiBox = {
    x: width / 6 - emojiSize / 2,
    y: height / 4 - emojiSize / 2,
    width: textWidth(emoji),
    height: emojiSize,
  };

  let emojiBox1 = {
    x: width / 6 - emojiSize / 2,
    y: height / 2 - emojiSize / 2,
    width: textWidth(emoji1),
    height: emojiSize,
  };

  // Check if mouse click is inside the bounding box of the first emoji
  if (
    mouseX > emojiBox.x &&
    mouseX < emojiBox.x + emojiBox.width &&
    mouseY > emojiBox.y &&
    mouseY < emojiBox.y + emojiBox.height
  ) {
    coupleAscii();
  }

  // Check if mouse click is inside the bounding box of the second emoji
  if (
    mouseX > emojiBox1.x &&
    mouseX < emojiBox1.x + emojiBox1.width &&
    mouseY > emojiBox1.y &&
    mouseY < emojiBox1.y + emojiBox1.height
  ) {
    singleAscii();
  }
}

I also intended to add an Adafruit Neopixel LED Strip through Arduino into my project in order to emphasise the experience of a Photo Booth. The LED strip would be attached to the screen and would turn on if the image classification model detected two people. As I was already experiencing issues with the code, I was unable to incorporate this into my final project despite attempts at wiring and programming it.

Improvements

Clearly, there is a lot of space for improvement within my final project. Other than improving the functionality of all the components I was unable to incorporate properly (or at all), one key area of improvement would have been testing and sharing my ideas with someone more experienced in order to gauge their feasibility. Had I known ml5 libraries tend to run slower in p5js, I may have chosen to pursue a different concept altogether.

Most importantly, this project is proof of the importance of simplifying ideas and starting to work on them ahead of time. The main issue that lead to this outcome was that I did not give myself enough time to understand these new concepts (image classification, ASCII) which naturally lead to me being unable to incorporate them successfully within a p5js sketch. Going forward, I will ensure that I set myself realistic goals and provide myself with a workable timeline by experimenting with ideas earlier on.