Concept:

I was fascinated by the idea of creating a game that could be controlled using hand movements, and I successfully brought that idea to life using p5.js and Arduino. Inspired by the game ‘Stick Hero,’ I decided to recreate it with a unique twist. In this game, the player’s objective is to achieve a high score by helping their character cross platforms using a growing stick. By clenching their fist, the player can extend the stick to bridge the gap between platforms. The challenge lies in finding the perfect stick length—not too short to fall short of the other platform, and not too long to overshoot it.

p5.js:

The p5.js code represents a game where players navigate platforms by using a growing and rotating stick.

1. Global variables are declared for images and sounds.
2. Images and sounds are preloaded to ensure smooth gameplay.
3. The Game class is defined, responsible for managing game objects and states.
4. The class includes functions for start checking, score display, platform removal, game over detection, state management, platform and stick creation, and game display.
5. The Player class represents the player character, incorporating functions for setting the destination, sticking to platforms, moving, and displaying.
6. The Platform class represents the platforms and includes functions for movement and display.
7. The Stick class represents the growing and rotating stick, featuring functions for growth and rotation.
8. The setup and draw functions are defined for game initialization and updates.
9. Event handlers for mouse clicks and releases are implemented to capture player interactions.
10. The code also includes a callback function for reading serial data from an Arduino, enabling hand-controlled gameplay.

The code creates an engaging game where players maneuver their character across platforms using a stick, striving to reach the end without falling off. The game dynamically changes based on mouse interactions and player movements, and the score is constantly displayed during gameplay.

p5.js Code:

// Declaring the global variables
let bg_img;
let player_img;
let success;
let failure;

// prelaoding the images and sounds
function preload() {
  start_screen = loadImage("start_screen.png");
  bg_img = loadImage("bg1.jpeg");
  player_img = loadImage("sprite.png");
  success = loadSound("success-sound-effect.mp3");
  failure = loadSound("failure.mp3");
}

// Setting up the canvas and creating the game object
class Game {
// Declaring the variables
  constructor() {
    this.platforms = [];
    this.gaps = [];
    this.widths = [];
    this.stick = false;
    this.state = 0;
    this.score = 0;
    this.start_x = 217;
    this.start_y = 352;
    this.start_w = 98;
    this.start_h = 30;
// Making an array of all the possible gaps between the platforms
    for (let g = 4; g < 61; g++) {
      this.gaps.push(g * 5);
    }

// Making an array of all the possible widths of the platforms
    for (let w = 8; w < 31; w++) {
      this.widths.push(w * 5);
    }

    // Making the first three platforms
    let x = 0;
    for (let i = 0; i < 2; i++) {
      let gap = random(this.gaps);
      this.create_platform(x);
      x = x + this.platforms[this.platforms.length - 1].w + gap;
    }

    // Making the player
    this.player = new Player(
      this.platforms[0].x + this.platforms[0].w,
      this.platforms[0].y
    );
  }

  // Function to display the game
  check_start() {
    if (
      mouseX >= this.start_x &&
      mouseX <= this.start_x + this.start_w &&
      mouseY >= this.start_y &&
      mouseY <= this.start_y + this.start_h
    ) {
      this.state = 1;
    }
  }

// Function to display the start screen
  display_score() {
    textSize(20);
    noStroke();
    fill(0);
    text("Score: " + this.score, 20, 30);
  }

// remove the platform once it is out of the screen
  remove_platform() {
    if (this.platforms[0].x <= -this.platforms[0].w) {
      this.platforms.shift();
    }
  }

  // Function to check if the game is over
  check_game_over() {
    if (this.state === 8 && this.player.x === this.player.destination_x) {
      failure.play();
      this.state = 9;
    }
  }

  // Function to display the game over screen
  display_game_over() {
    background(bg_img);
    textSize(20);
    noStroke();
    fill(0);
     textSize(30);
    text("Score: " + this.score, width / 2 - 50, height / 2-50);
    text("Game Over", width / 2 - 70, height / 2);
    text("Click to restart", width / 2 - 88, height / 2 + 50);
  }

 state_manager() { // Function to manage the states of the game

    if (this.state === 0) { // State 0 displays the game instructions.
      image(start_screen,0,0);
    } else if (this.state === 2) { // State 2 detects a mouse click and grows the stick.
      this.create_stick();
      this.stick.grow();
    } else if (this.state === 3) { // State 3 detects a mouse release and rotates the stick.
      this.stick.rotate();
    } else if (this.state === 4) { // State 4 checks if the player has reached the platform after the stick has finished rotating, and transitions to state 5 or state 8 accordingly.
      this.player.set_destination();
    } else if (this.state === 5) { // State 5 moves the player towards their destination.
      this.player.move();
    } else if (this.state === 6) { // State 6 determines the new positions of the platforms and the player once the player reaches their destination platform.
      this.set_platforms_destination();
    } else if (this.state === 7) { // State 7 moves the platforms and the player towards their destination.
      for (var i = 0; i < this.platforms.length; i++) {
        this.platforms[i].move();
        this.player.stick_to_platform();
      }
      this.remove_platform(); 
    } else if (this.state === 8) { // State 8 moves the player towards the end of the stick and checks if they have reached it.
      this.player.move();
      this.check_game_over();
    } else if (this.state === 9) { // State 9 ends the game and displays the game over screen as soon as the player reaches the end of the stick that is not on the platform.
      this.display_game_over();
    }
  }

 // Function to create a new platform
 create_platform(x) {
    let w = random(this.widths);
    let y = height - 100;
    let p = new Platform(x, y, w, 100);
    this.platforms.push(p);
  }

// Function to set the destination of the platforms
  set_platforms_destination() {
    this.create_platform(width);
    this.platforms[0].destination_x = -this.platforms[0].width;
    this.platforms[1].destination_x = 0;
    this.platforms[2].destination_x = this.platforms[1].w + random(this.gaps);
    game.state = 7;
  }
    
// Function to create a new stick
  create_stick() {
    if (!this.stick) {
        this.stick = new Stick(
          this.platforms[0].x + this.platforms[0].w, this.platforms[0].y,3,0);
      }
  }

// Function to display the game
  display() {
    background(bg_img);
    this.state_manager();
    if (game.state != 9 && game.state != 0) {
      this.display_score();
      for (let j = 0; j < this.platforms.length; j++) {
        this.platforms[j].display();
        this.player.display();
        if (this.stick != false) {
          this.stick.display();
        }
      }
    }
  }
}

// class for player
class Player {
  constructor(x, y) {
    this.w = 30;
    this.h = 50;
    this.x = x - this.w;
    this.destination_x = x;
    this.v = 5;
    this.y = y - this.h;
    this.position = 0;
  }

  // Setting Destination of the player so that he moves after the stick is down
  set_destination() {
    if (
      game.stick.x_2 >= game.platforms[1].x &&
      game.stick.x_2 <= game.platforms[1].x + game.platforms[1].w
    ) {
      this.destination_x = game.platforms[1].x + game.platforms[1].w - this.w;
      game.score += 1;
      success.play();
      game.state = 5;
    } else {
      this.destination_x = game.stick.x_2;
      game.state = 8;
    }
  }

  // Setting player's x equal to the platform so it moves along with it
  stick_to_platform() {
    if (game.platforms.length === 2) {
      this.x = game.platforms[0].x + game.platforms[0].w - this.w;
      if (game.platforms[0].x === 0) {
        game.state = 1;
      }
    } else {
      this.x = game.platforms[1].x + game.platforms[1].w - this.w;
      if (game.platforms[1].x === 0) {
        game.state = 1;
      }
    }
  }

  // Function to move the player according to his destination
  move() {
    if (this.x < this.destination_x) {
      this.x += this.v;
      this.position = (this.position + 1) % 7;
    } else if (this.x > this.destination_x) {
      this.x -= this.v;
    } else if (
      this.x == this.destination_x &&
      this.x > game.platforms[0].x + game.platforms[0].w &&
      game.state === 5
    ) {
      game.stick = false;
      game.state = 6;
    }
  }

  // Display the player using the respective position from the sprite sheet
  display() {
    let c = player_img.get(this.position * 109, 0, 109, 120);
    image(c, this.x, this.y, this.w, this.h);
  }
}

// Declaring the platform class
class Platform {
  constructor(x, y, w, h) {
    this.x = x;
    this.destination_x = x;
    this.y = y;
    this.w = w;
    this.h = h;
    this.v = 5;
  }

  // Function to move the platform according to its destination
  move() {
    if (this.x != this.destination_x) {
      this.x = this.x - this.v;
    }
  }

  // Display the platform
  display() {
    noStroke();
    fill(color("#808080")); //"#6d4a3b"
    rect(this.x, this.y, this.w, this.h);
    stroke(0);
  }
}

// Declaring the Stick class
class Stick {
  constructor(x, y, w, h) {
    this.l = h;
    this.x_1 = x;
    this.y_1 = y;
    this.x_2 = x;
    this.y_2 = this.y_1 + this.l;
    this.angle = PI / 2;
  }

  // Function to grow the stick
  grow() {
    this.y_2 -= 5;
    this.l += 5;
  }

  // Rotate the Stick according when the mouse if released and check if the rotation is complete
  rotate() {
    this.angle -= PI / 64;
    this.x_2 = this.x_1 + this.l * cos(this.angle);
    this.y_2 = game.platforms[0].y - this.l * sin(this.angle);

    if (this.angle <= 0) {
      game.state = 4;
    }
  }

  // Display the stick
 display() {
  stroke("#808080"); 
  strokeWeight(2);
  line(this.x_1, this.y_1, this.x_2, this.y_2);
  strokeWeight(4);
  
}
}

function setup() {
  createCanvas(550, 450);
  game = new Game();
}

function draw() {
  clear();
  game.display();
}

// Perform functions when mouse is clicked according to the state of the game
function mousePressed() {
  if (game.state === 0) {
    game.check_start();
  } else if (game.state === 1) {
    game.state = 2;
  } else if (game.state === 9) {
    game = new Game();
    game.state = 1;
  }

}

function readSerial(data) //call back function
{
  if (data != null) //if the data received is not null
    {
      console.log(data);
      if (game.state === 0) {
    // game.check_start();
  } else if (game.state === 1 && data > 1008) {
    game.state = 2;
  } else if (game.state === 9) {
    // game = new Game();
    // game.state = 1;
  }
    else if( game.state ===2 && data < 1008)
    {
       game.state = 3;
    }
    
     
    }
  
  let redlight= 2;
  let greenlight = 1;
    if(game.state === 2) 
    {
      let sendToArduino = 1 + "\n";
      writeSerial(sendToArduino);
    }
    else if(game.state === 9) 
    {
      let sendToArduino = 2 + "\n";
      writeSerial(sendToArduino);
    }
    else
    {
      sendToArduino = 0 + "\n";
      writeSerial(sendToArduino);
    }
  
}

function keyPressed() //if any key is pressed, then set up serial
{
  setUpSerial();
}

// Shift the state when the mouse is released
function mouseReleased() {
  if (game.state === 2) {
    game.state = 3;
  }
}

Arduino:

The code reads analog input from a flex sensor connected to pin A4 and controls two LEDs (green and red) connected to pins 8 and 2, respectively.

In the `loop()` function:

– The flex sensor’s analog value is read using `analogRead()` and stored in the `value` variable.

– The analog value is printed to the serial monitor using `Serial.println()`.

– If input is available from p5.js, the code reads the value and checks for specific conditions.

– Based on the brightness value, the green and red LEDs are controlled by turning them on or off using `digitalWrite()`.

The code utilizes analog input from a flex sensor to control the brightness of two LEDs connected to pins 8 and 2, based on input received from p5.js via serial communication.

Arduino Code:

//Constants:

const int flexPin = A4; // Pin A4 to read analog input
const int ledPin = 8; // Green LED pin
const int ledPin2 = 2; // Red LED pin

// Variables:
int value; // Save analog value

void setup() {
  Serial.begin(9600); // Begin serial communication
  pinMode(ledPin, OUTPUT);
  pinMode(ledPin2, OUTPUT);
}

void loop() {
  value = analogRead(flexPin); // Read analog input from flex sensor
  Serial.println(value); // Print the analog value to the serial monitor
  delay(100); // Small delay

  // Wait for input from p5.js
  while (Serial.available()) {
    int brightness = Serial.parseInt();

    if (Serial.read() == '\n') {
      // Control the LEDs based on the received brightness value
      if (brightness == 1) {
        digitalWrite(ledPin, HIGH);
        brightness = 0;
      } else {
        digitalWrite(ledPin, LOW);
      }

      if (brightness == 2) {
        digitalWrite(ledPin2, HIGH);
        brightness = 0;
      } else {
        digitalWrite(ledPin2, LOW);
      }
    }
  }
}

User Testing:

Improvements:

Difficulty Progression: Enhance the gameplay experience by implementing a progressive difficulty system. As the player progresses, introduce challenges such as faster platform movement, shorter time limits to place the stick, or additional obstacles. This will keep players engaged and provide a sense of accomplishment as they overcome increasingly difficult levels.

Power-ups: Introduce exciting power-ups or bonuses that players can collect during gameplay. These power-ups could temporarily slow down platform movement, extend the stick’s length, grant extra lives, or introduce other unique abilities. Power-ups add depth, strategy, and an element of surprise to the game, making it more enjoyable and rewarding.

What I am proud of:

I am proud in successfully bringing my initial concept to life. Creating a game that can be controlled by hand movements is truly amazing. I also quite like the game dynamic as it is very visually appealing. It has been a joy to see the enjoyment that people experience while playing the game, as it has been positively received during testing with various individuals.

Project ‘Pakistani Moseeqi’ (translates to Pakistani music)

Description: A Sound Visualizing Musical Instrument

Concept and corresponding elements

A sound visualization instrument that allows you to:

1. 1. mix and play sounds to make your own unique music and 
   2. visualize it on the p5js screen

An ode to Pakistani music

This project pays tribute to Pakistani music by incorporating sounds from instruments that are unique to Pakistani music such as the tabla, bansuri, sitar, and veena among others. This way, I wanted to expose the NYUAD community, who will use my project, to Pakistani music while also paying tribute to my culture. 

Accessibility

With buttons, this instrument gives one the opportunity to make their own music. To make it accessible to people who are blind/visually impaired, I used a simple design with an easy-to-use interface that does not require many instructions to read. This way, someone who cannot see can still enjoy it and make their music. 

Additionally, to make it accessible to people who cannot hear, I added the sound visualization element so that they can see a unique visualization of each sound and hence, enjoy the instrument in a different way – as a visualization tool. 

Interaction design, feedback, and implementation

• • 4 momentary LED-Buttons to play 4 different sounds when pressed
• 
  • Each button has an LED light that turns off when the button is pressed to indicate to the user that it is working
  • Each button plays a different sound when pressed. 
  • When pressed, button sends a signal to the p5js to display the animation/visualization associated with that button and sound
  • More than one button can be pressed simultaneously.
  • Turned on/off indicator led light
  • The blue ‘power’ LED on the bottom left is an indicator. When the instrument is in use, it is turned on. When not, it is off. This way, it provides indication to the user when the instrument is working and when it is not. 
  • Arduino/physical elements – contained within a cardboard box
  • 5th non momentary button/switch – this button is not momentary – it, when pressed, plays a background music. The user can then add more sound effects on top of this music. When pressed again, the button stops the music. And this can be repeated. 

P5js elements:

• • Start page – this page displays the cover and brief instructions to keep everything simple and easy to use
  • The play button leads to the music creation and sound visualization page – here, whenever a physical button is pressed, a visualization associated with the sound appears while the button is pressed
  • A slider to control volume of the sound effects from the momentary buttons (not the background music – so that the user controls the volume of the sound they “add” to the background music.
  • Inspiration: https://patatap.com/ 

Arduino Code

For the Arduino end, as mentioned, buttons and LEDs are arranged in a box. 

Inputs and Outputs:

• • From LED buttons to Arduino, which process and send information to p5 

From P5: 

• • Indication if instrument turned on/off: power LED on/off
  • LEDs to indicate if instrument is on or if any button is pressed (feedback)

const int redButtonPin = A2;
const int greenButtonPin = A1; 
const int yellowButtonPin = A3;
const int blueButtonPin = A4;
const int ledPin = 8;
const int songButtonPin = 7; 

void setup() {
  // put your setup code here, to run once:
  Serial.begin(19200);
  //pinMode(blueLedPin, OUTPUT);
  pinMode(redButtonPin, INPUT);
  pinMode(greenButtonPin,INPUT);
  pinMode(yellowButtonPin,INPUT);
  pinMode(blueButtonPin,INPUT);
  pinMode(ledPin, OUTPUT);
  pinMode(songButtonPin, INPUT);

   // start the handshake
  while (Serial.available() <= 0) {
    digitalWrite(LED_BUILTIN, HIGH); // on/blink while waiting for serial data
    Serial.println("0,0"); // send a starting message
    delay(300);            // wait 1/3 second
    digitalWrite(LED_BUILTIN, LOW); 
    delay(50);
    digitalWrite(ledPin, LOW);
  }
}

void loop() 
{
  // put your main code here, to run repeatedly:
  while (Serial.available())
  {
    digitalWrite(LED_BUILTIN, HIGH);
    //if red button pushed
    
    int redButton = digitalRead(redButtonPin);
    int greenButton = digitalRead(greenButtonPin);
    int yellowButton = digitalRead(yellowButtonPin);
    int blueButton = digitalRead(blueButtonPin);
    int songButton = digitalRead(songButtonPin);
    Serial.print(redButton);
    Serial.print(",");
    Serial.print(greenButton);
    Serial.print(",");
    Serial.print(yellowButton);
    Serial.print(",");
    Serial.print(blueButton);
    Serial.print(",");
    Serial.println(songButton);
    //delay(100);


    int light = Serial.parseInt();
    if (light == 1)
    {
      digitalWrite(ledPin, HIGH);
    }
    if (light == 0)
    {
      digitalWrite(ledPin, LOW);
    }

  }
}

P5 Sketch and Code

The P5 screen will first present very brief instructions on how to use the instrument. Once the experience begins, the screen will show a visualization of the music created using input from the sensors. 

Inputs and outputs:

• • Use inputs from buttons that Arduino sends to draw shapes/patterns on the screen
  • Display details such as volume/start instruction on the screen

let incrementCircle = true;
let greenB=true;
let blueB=true;
let yellowB=true;
let playBg1 = true;
let playStart = true;
let start = true;
let numOfCircles = 0;
let circleX = [];
let circleY = [];
let circleWidth = []
let s = 0;
let alt = true;
let mode = 'pause';
let drum, flute, sitar, veena; 
let volumeSlider; 
let vol;
function setup() {
  startSound = loadSound('audio/bubbles.mp3');
  starting = loadSound('audio/start.mp3');
  volumeSlider = createSlider(0,100);
  volumeSlider.position(650, 20);
  drum = loadSound('audio/tabla.mp3');
  flute = loadSound('audio/bansuri.mp3');
  sitar = loadSound('audio/strike.mp3');
  veena = loadSound('audio/spiral.mp3');
  bgMusic = loadSound('audio/calmBg.mp3');
  //drumA=createAudio('audio/drum.wav');
  startImg = loadImage('images/pak music.png');
  displayImg = loadImage('images/display.png');
  startMusic = loadSound('audio/clay.mp3');
  createCanvas(800, 800);
  
  angleMode(DEGREES);
  rectMode(CENTER);
  
  
} 

function draw() {

  vol = volumeSlider.value();
  drum.setVolume(vol);
  flute.setVolume(vol);
  veena.setVolume(vol);
  sitar.setVolume(vol);
  
  if (mode == 'pause')
  {
    startPage();
    
  }
  
  
  if (mode == 'play')
    {
      background(0);
      for (i = 0; i < 10; i++)
      {
      fill('#F44336A5');
      //CAN DRAW PERMANENT CIRCLES
      noStroke();
      ellipse(width/2, height/2, s+15*i, s+15*i);
      fill('rgb(126,9,9)');
      ellipse(width/2, height/2, s-i*10, s-i*10);
      }
      s = s+10;
      if (playStart == true){
           startSound.play();
        playStart = false;
      }
   
    }
    if (mode == 'drums')
    { //if button pressed, random circles 
      if (incrementCircle == false)
      { 
        playDrums();
      }
      else 
      {
        drawCircles();
      }
    }
      
      
    if (mode == 'sitar')
      {
         if (blueB == false )
          {
            background(0);
          }
        else if (blueB == true)
          {
            playSitar();
          }
      }
  
      if (mode == 'flute')
       {
         console.log(mode);
         console.log(greenB);
        if (greenB == false)
          {
            background(0);
            playFlute();

          }
        else if (greenB == true)
          {
            console.log('calling playFlute')
            playFlute();
          }
      }
      if (mode == 'veena')
      {
        if (yellowB == false)
        {
          background(0);
        }
        else if (yellowB==true)
        {
          playVeena();
        }
      }
      
  textSize(12);
      if (!serialActive) 
      {
        text("Press Space Bar to select Serial Port", 20, 30);
      } else 
      {
        text("Connected", 20, 30);
        let volumeLabel = "Volume: " + vol;
        text(volumeLabel, 600, 20 );
        //everything else goes here

      }
    
}

function readSerial(data) {
    ////////////////////////////////////
    //READ FROM ARDUINO HERE
    ////////////////////////////////////
     if (data != null) 
     {
      //console.log("data:");
       console.log(data);
       // make sure there is actually a message
       let fromArduino = split(trim(data), ",");
       // if the right length, then proceed
       if (fromArduino.length == 5) 
       {
         let redButton = int(fromArduino[0]);
         if (redButton == 1)
         {
           mode = 'drums';
           if (incrementCircle == true)
           {
             numOfCircles = numOfCircles + 1;
             circleX[numOfCircles] = random(width);
             circleY[numOfCircles] = random(height);
             circleWidth[numOfCircles] = random(100);
             incrementCircle = false;
             drum.play();
           }
         }
         else if (redButton == 0)
           {
             incrementCircle = true;
           }
         
         let greenButton = int(fromArduino[1]);
         
         if (greenButton == 1)
         {
           mode = 'flute';
           if(greenB==true)
           {
             console.log("greenb set to false");
             flute.play();
             greenB = false;
           }
         }
        else if (greenButton == 0)
        {
            greenB = true;
            console.log("greenb set to true");
        }
         
         let yellowButton = int(fromArduino[2]);
          if (yellowButton == 1)
         {
           mode = 'sitar';
           if (yellowB == true)
           {
             sitar.play();
             yellowB=false;
           }
         }
         else if (yellowButton == 0)
           {
             yellowB = true;
           }
        
         let blueButton = int(fromArduino[3]);
           if (blueButton == 1)
         {
           mode = 'veena';
           if (blueB == true)
           {
             veena.play();
             blueB=false;
           }
         }
         else if (blueButton == 0)
           {
             blueB = true;
           }
         
         let songButton = int(fromArduino[4]);
         if (songButton ==1)
           {
             console.log('here');
             if (playBg1 == true && mode == 'play')
               {
                 console.log('here1');
                 bgMusic.play();
                 playBg1 = false;
               }
           }
          else if (bgMusic.isPlaying() == true && mode == 'play' && songButton == 0)
               {
                 console.log('here2');
                 bgMusic.stop();
                 playBg1 = true;
               }
      
         
    if (greenButton == 0 && blueButton ==0 && yellowButton == 0 && redButton == 0 && mode != 'pause')
           
      {
        mode = "play";
      }         
         
  // if (playBg1==true && mode == 'play')
  //   {
  //     startMusic.play();
  //     playBg1 = false;
  //   }
  // else {
  //   //calmBg.stop();
  // }         
         /////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////
       }
          //////////////////////////////////
          //SEND TO ARDUINO HERE (handshake)
          //////////////////////////////////
       let light; 
      if (mode != 'pause')
      { 
        light = 1;
      }
       else if (mode =='pause')
         {
           light =0;
         }
        let sendToArduino = light + "\n";
        writeSerial(sendToArduino);
     }
}

function startPage()
{
  
  //background('white');
  image(startImg, 0, 0, width, height);
  //fill(255);
  // textSize(64);
  // text("موسیقی",60,120);
  // textSize(20);
  // fill(0);
  // rect(width/3-10,height-50,200,60);
  
  fill(255);
  let x = mouseX;
  let y = mouseY;
  let t = x+','+y;
  text(t, mouseX, mouseY);
  //text("Press 's' to play your own music", width/3-230, height-10); 
  textSize(14);
      textStyle(NORMAL);
      text("Play, mix, and visualize Pakistani music by pressing the buttons", width/4, 3*height/4+50);
  if (mouseX >= 242 && mouseX <=558 && mouseY >=679 && mouseY <= 764)
    {
      noStroke();
      fill('#503418')
      ellipse(390,720,120,75);
      fill(255);
      textSize(38);
      //fontWeight(200);
      textStyle(BOLD);
      text("PLAY",335,734);
      
      if (mouseIsPressed)
        {
          mode='play';
        }
    }
  
}

function keyPressed() 
{
  if (key == " ") 
  {
    // important to have in order to start the serial connection!!
    setUpSerial();
    
    if (playStart==true && mode == "play")
    {
      startMusic.play();
      playStart = false;
    }
    
    startMusic.play();
  }   
  if (key == 's')
  {
    mode = "play";
  }
//   if (key =='d')
//     {
//       mode = 'drums'
//       incrementCircle = false;
//       drum.play();
//     }
// if (key == 'f')
//     {
//       mode = 'flute';
//       flute.play();
//     }
//   if (key == 'g')
//     {
//       mode = 'sitar';
//       sitar.play();
//     }
//   if (key == 'v')
//     {
//       mode = 'veena';
//       veena.play();
//     }
  
  
}

function drawCircles()
{
  noStroke();
  //fill(random(255));
  fill(240);
  for (i =0; i < numOfCircles; i++)
    {
      circle(circleX[i], circleY[i], circleWidth[i]);
    }
}

function playDrums()
{ 
  if (alt== true)
    {alt = false}
  else
  {alt =true;}
 
   background('black');
  for (i = 23; i > 0; i--)
    {
      noStroke();
      if (alt == true)
      {
        if(i%2 == 0)
        { 
          fill('white');
        }
        else 
        {
          fill('black');
        }
      }
      else 
        {
          if(i%2 == 0)
        { 
          fill('black');
        }
        else 
        {
          fill('white');
        }
        }
      ellipse(width/2, height/2, random(50*i));
    }
   fill(0);
  //ellipse(width/2,height/2,40)
}

function playFlute()
{
  console.log("here ");
  background(0);
  noFill();
  push();
  translate(width/2, height/2);
  for (let i =0; i < 200; i++)
  {
    push();
    rotate(sin(frameCount+i)*100);
    
      let r = map(sin(frameCount), -1,1,50,255);
      let g = map(cos(frameCount/2),-1,1,50,255);
      let b = map(sin(frameCount/4),-1,1,50,255);
    noFill();
    stroke(r,g,b);

    rect(0,0,800-i*2,300-i);
    
    pop(); 
  }
  pop();
}

function playSitar()
{
  noStroke();
  for (i =30; i>0; i--)
 {
   if (i%2==0)
   {
     fill(0);
   }
   else
    {
     fill(255,150,0);
    }
   square(width/2, height/2, random(50*i));
 }
}
let direction = 1;

function playVeena()
{
  background(0);
  noStroke();
  let x=random(width);
  let y=0;
  direction = random(-1,1);
    //random(0,width);
  
  //y=random(0,height);
  for (i =0; i < 3*width;i++)
    {
      x=x+direction;
      y=y+1;
      fill(random(x,x+30),random(y-20,y+10),random(y-30,y+20));
      circle(x,y,30);
    }
}

function displayScreen()
{
  image(displayImg, 0,0, width, height);
}

User testing

For the most part, the design was simple to understand for the users. The simple instruction to use buttons to play music for users to understand how to begin. Then, as they pressed each, they easily saw its connection to sound and the visualization on the p5 and used this information to improvise and process the information.  

Things that can be improved: 

The slider for volume is not as visible or clear so we can replace it with clearer labels in the future.

More buttons can be added to create an array of different sounds.

More western music can ALSO be integrated to add a cultural diversity angle to the project.

Bigger box can be used so wires and Arduino can easily be places inside

The 5th red button was, in a way, ignored by the user due to its small size and because it looked small and unimportant alongside all the led lights and buttons. This button can be replaced with something better. 

What I am proud of:

I am particularly proud of my sound visualizations. They add a dynamism and motion to the project that makes listening to each sound even more fun and interesting. The visualizations are very appealing to the eye. I am also proud that I attempted to make the instrument more accessible as this reinforced the importance of keeping accessibility in mind when designing things. 

Main Concept

For the final project, I created a California themed car game. The physical hardware consists of a box with an arduino installed inside along with an ultrasonic sensor. There is a long road panel in front of it where the player places their NYU card keys and slides it along the road lanes to mimic the movement of a car on the road. The ultrasonic sensor detects the motion of the card along the lanes and causes the actual car in p5.js to move as well. On the screen, I decided to design the UI on a californian theme so I went with pictures and elements which depicted that concept. When the game begins, there are cars approaching the player from each lane and the player needs to move their key on the physical road to change the lane of the car on screen. With time the car speed increases and becomes more difficult. As it is a high score based game, every incoming car that is avoided is counted as one point so the ending page shows the high score and individual player score as well. As an indication of when the game has started, the green LED lights up on the box labeled “gas station” and similarly when the car crashes, red LED lights up or yellow goes off when the player is reading instructions, almost to imitate real life traffic lights. 

P5.js

In the p5.js code, I have created a game using different classes. There is the player class which deals with the movement of the player by receiving values from arduino and then mapping it according to the canvas and player position. Then there is a class for the incoming cars, which deals with the downward motion of the cars and is also passed as an argument to the player class to check if the x or y coordinates of the player coincide with those of the incoming cars, in which case the game ends as the cars crash. I created an array for the incoming cars so that they are added at a certain frameRate count and are removed from the array as soon as they go out of the canvas.

//variable declarations


let h = 600;
let w = 400;
let count = 0;
let pos = [55, 165, 270];
let charsize = 80 ; //defines size of player
let charheight = 100
let player; //variable for the player class object
let newcarssize = 100; 
let newcarswidth = 80;
let newcars = [];
let score=0;
let highscore=0;
let bonusarr = []; //array to store bonus objects
let caught = false; 
let bonuscaught = []; //
let scorebool = false;
let newchar = false; //boolean variable to help in creating new character when game starts again
let foodnum = 0; //to iterate over food array
let speedincr = 1; //for the vertical motion of newcarss
let carimg = 0;
let carspeeds = 2;

let r1=0;
let r2 =-600;
let r3 = 0;

let food = []; //array for food images

let gamestate = 'start'; //gamestate for changes in display
framewidth = 20; 

let a = 150;
let b = 300;
let c= 450;
let d = 550;

//image variables
var gif_loadImg, gif_createImg;
let pixelfont;
let startimg;
let startbutton;
let instructbutton;
let instructpage;
let endimg;
let road1, road2, road3;
let car1, car2, car3, car4, car5;
let cars = [];
let endsound, crash, backsound;
let tree;

//arduino variables
let rVal = 0;


//preload function for all the images and sound
function preload() {
    startimg = loadImage('2.png');
    startbutton = loadImage('startbutton.png');
    instructpage = loadImage('3.png');
    instructbutton = loadImage('instruct.png');
    endimg = loadImage('1.png');
    road1 = loadImage('road.png');
    road2 = loadImage('road.png');
    road3 = loadImage('road.png');
    car1 = loadImage('car1.png');
    car2 = loadImage('car2.png');
    car3 = loadImage('car3.png');
    car4 = loadImage('car4.png');
    car5 = loadImage('car5.png');
    tree = loadImage('tree.png');
  
    endsound = loadSound('endsound.mp3');
    crash = loadSound('crash.mp3');
    backsound = loadSound('background.mp3');
    

}




//====================================================
//class for the player object
class Character{
  constructor(){
    this.x = 5+ width/2-(charsize/2);
    this.y = height - ((charsize));
    this.speed = 4; //speed for the movement of player horizontally

  }
  
  display(){

    image(car2, this.x, this.y-10, charsize, charheight ); 


  }
  moveleft(){ //if left key is pressed, causes player to move left
    this.x -= this.speed;

    if (this.x < 0) {
      this.x = 0; //so player does not go beyond the canvas
    }
  }
  moveright(){ //if right key is pressed, causes player to move left
    this.x+=this.speed;

    if (this.x+charsize > width) {
      this.x = width-charsize; //so player does not go beyond the canvas
    }
  }
  update(newcars){
    
    if(rVal < 7){
      this.x = 55;
    }
    else if(rVal>=7 && rVal<14 ){
      this.x = 165;
    }
    else if(rVal >= 14){
      this.x = 270;
    }
    

    


        if(this.x>=newcars.x && this.x <= newcars.x+(newcars.pwidth) && this.y>=newcars.y &&  this.y<=newcars.y+ newcars.pheight){
          count++;

      console.log("Loss:" , count);
          crash.play();
          gamestate = 'over';
        

    }
    else if(this.x+charsize >=newcars.x && this.x+charsize <= newcars.x+(newcars.pwidth) && this.y>=newcars.y &&  this.y<=newcars.y+ newcars.pheight){
      count++;

      console.log("Loss:" , count);
      crash.play();
      gamestate = 'over';
            
            }
    
    else if(this.x>=newcars.x && this.x <= newcars.x+(newcars.pwidth) && this.y+charheight>=newcars.y &&  this.y+charheight<=newcars.y+newcars.pheight){
      
          count++;
      crash.play();

          console.log("Loss:" , count);
      gamestate = 'over';
            }

    

  }





}
//====================================================
//class to create newcarss
class incars{
  constructor(carimg, carspeeds){
    this.x = random(pos);
    this.y = 0;
    this.pwidth = newcarswidth;
    this.pheight = newcarssize;
    this.speed = carspeeds;
    this.num = carimg
    
  }
  
  displaynewcars(){
 
     image(cars[this.num], this.x, this.y , this.pwidth, this.pheight);
  }


  //causes newcars to move upwards
  update(){
    this.y += this.speed;

  }
  
  
}


function setup() {
  
  createCanvas(w,h);
  
  background(220);
  player = new Character();
  cars = [car1, car2, car3, car4, car5];





}

function draw() {
  if(frameCount == 1){
    backsound.play();
  }
  background(220);
   if (!serialActive) 
  {
    text("Press Space Bar to select Serial Port", 20, 30);
  } else 
  {

  }



  
  if(gamestate == 'start'){ //if game is at start page then display homepage


    startpage();
    



 
    
  }
  //if gamestate is of 'game' then plays the game through thegame() function
  else if(gamestate == 'game'){
        image(road1,  0,r1, width, height, 0, 0, road1.width, road1.height); //homepage image displayed
    image(road1, 0,r2,  width, height, 0, 0, road1.width, road1.height);
    
    r1+=2;
    r2+=2;

    if(r1 >= height){
      r1 = r2- height ;
    }
    if(r2>= height){
      r2 = r1-height ;
    }

 
    
    
    
  thegame(); //plays game through this function
    a++;
    b++;
    c++;
    d++;
    
       image(tree,  -75,a, 150, 150, 0, 0, tree.width, tree.height);
        image(tree,  -75,b, 150, 150, 0, 0, tree.width, tree.height);
            image(tree,  width-60,c, 150, 150, 0, 0, tree.width, tree.height);
        image(tree,  width-60,d, 150, 150, 0, 0, tree.width, tree.height);
    
    if(a > height+150){
      a = -150;
    }
    else if(b> height+150){
      b = -150;
    }
    else if(c> height+150){
      c = -150;
    }
        else if(d> height+150){
      d = -150;
    }
    

    


    


  }
  
  else if(gamestate == 'over'){ //if gamestate is over then end page displayed 


    image(endimg, 0, 0, width, height, 0, 0, endimg.width, endimg.height);

  endpage();
   
    
  }
  else if(gamestate == 'instructions'){ //displays instructions 
    instructionspage();

  }
}
//========================
function thegame(){ 

 
   //checks if game is started again and creates a new player to play the game again
    if(newchar == true){
     player = new Character();
    newchar = false;
  }
  
  
  


  //for keys to move the player
  if(keyIsDown(LEFT_ARROW)){
    player.moveleft();
  }
  else if (keyIsDown(RIGHT_ARROW)){
    player.moveright();
  }
  


  
  //for new newcars and elements to be created and added to their arrays
  if(frameCount%150 == 0){

    newcars.push(new incars(carimg, carspeeds));

    score++;
    console.log(score);
    carimg++;
    if(carimg>4){
      carimg = 0;
    }
    
 
  }


  
  //FOR LOOP
  for(let i = 0 ; i<newcars.length ; i++){
    //shows newcarss and updates their positions by decreasing y coordinated to move them up
    newcars[i].displaynewcars();
    
    newcars[i].update();
     
    

    

    //when newcars and food goes beyond the roof, they are removed from their arrays 
    if (newcars[i].y > height+400 ){
      newcars.splice(i, 1);
      // bonusarr.splice(i, 1);
      // bonuscaught.splice(i,1);
 

    }
    
    //Player's position checked against newcars and ladder's position
    player.update(newcars[i]);

    newcars[i].update();//moves it up
    
    
  }
  
  player.display(); 
  
  //END of FOR
  
  
  


  
}

//==============================

function startpage(){ //function to display the images of starting page
    image(startimg, 0, 0, width, height, 0, 0, startimg.width, startimg.height);
  


    imageMode(CENTER);
      image(startbutton, width/2, height/2, map(startbutton.width, 0, startbutton.width, 0, width/4), map(startbutton.height, 0, startbutton.height, 0, height/6));

    imageMode(CORNER);

    //checks if mouse pressed on the button and directs to instructions
    if(mouseX < width/2 + (width/(width/100)/2) && mouseX > width/2 - (width/(width/100)/2) && mouseY > height/2 - (height/(height/100)/2) && mouseY < height/2 + (height/(height/100)/2) && mouseIsPressed){
      gamestate = 'instructions';
      // success.play();
    }
  
}



function endpage(){ //function to display score and highscore and 'game over' page


  for(let i = 0 ; i<newcars.length ; i++){
      newcars.pop();
      bonusarr.pop();
      bonuscaught.pop();
  }
  newchar = true;
  
  if(score>highscore){ //updates the highscore with each previous score 
    highscore = score;
  }

  textAlign(CENTER);
  textFont('Times New Roman');
  fill("#FAF8F4");
  textSize(0.05*width);
  text('Score: '+score , width/2, (height/2)-58);
  text('High Score: '+highscore , width/2, ((height/2))-40);

  
  if(keyIsDown(ENTER)){ //when ENTER is pressed, gamestate changed so that game starts again
    gamestate = 'start';
    score= 0 ;
  }

}

function instructionspage(){ //displays instruction images 
  
  image(instructpage, 0, 0, width, height, 0, 0, instructpage.width, instructpage.height);
  

  image(instructbutton, (2.3*width)/4, (3.3*height)/3.7, map(instructbutton.width, 0, instructbutton.width, 0, 150), map(instructbutton.height, 0, instructbutton.height, 0, 50));
  
  //checks if mouse clicked on the button and continues to game state of playing the game
  if(mouseX>(2.3*width)/4 && mouseX < width && mouseY > (3.3*height)/3.7 && mouseY < height && mouseIsPressed){
    gamestate = 'game';
    //success.play();
  }

  

}


//====================
//ARDUINO



function keyPressed() {
  if (key == " ") 
  {
    // important to have in order to start the serial connection!!
    setUpSerial();
  }
}
function mouseIsPressed()
{
  readSerial();
}

// This function will be called by the web-serial library
// with each new line of data. The serial library reads
// the data until the newline and then gives it to us through
// this callback function
function readSerial(data) {
  ////////////////////////////////////
  //READ FROM ARDUINO HERE
  ////////////////////////////////////

  if (data != null) 
  {
    rVal = data;
  }
  
  //=============
  let sendToArduino=0;
  if(gamestate == 'game'){
    sendToArduino = 1;
  }
    else if(gamestate == 'instructions' || gamestate == 'start'){
    sendToArduino = 2;
  }
      else if(gamestate == 'over'){
    sendToArduino = 3;
  }
  sendToArduino = sendToArduino+"\n";
    
    writeSerial(sendToArduino);
}

Arduino

In arduino, the main component is the ultrasonic sensor and then the LEDs. I have set the trig and echo pins as well as the LED pins initially. The connection between arduino and p5.js is in sending data from the ultrasonic sensor when the card is moved in front of it. The data is read by p5.js and is then used to control the x-coordinate of the player car. The p5.js sends the data regarding the game states which controls the LED pins. The data is sent as an integer, each indicating the game state, and is then used to control the respective LEDs. I soldered the LEDs so I could fix it on top of the box. 

I was facing an issue with consistent readings from the ultrasonic sensor. As it sends out signals in a more spread out manner and reads very minor changes, the car was glitching because slightest tilt of the card was also causing the ultrasonic sensor to change values. I had to add a bit more delay to make sure the card movement has stabilized before the sensor reads its distance.

const int trigPin = 9;
const int echoPin = 10;
const int ledgreen = 2;
const int ledyellow= 4;
const int ledred = 7;
// defines variables
long duration;
int distance;
void setup() {
  pinMode(trigPin, OUTPUT); // Sets the trigPin as an Output
  pinMode(echoPin, INPUT); // Sets the echoPin as an Input
  Serial.begin(9600); // Starts the serial communication
  pinMode(ledgreen, OUTPUT);
  pinMode(ledyellow, OUTPUT);
  pinMode(ledred, OUTPUT);
}
void loop() {
  // Clears the trigPin
  digitalWrite(trigPin, LOW);
  delayMicroseconds(2);
  // Sets the trigPin on HIGH state for 10 micro seconds
  digitalWrite(trigPin, HIGH);
  delayMicroseconds(10);
  digitalWrite(trigPin, LOW);
  // Reads the echoPin, returns the sound wave travel time in microseconds
  duration = pulseIn(echoPin, HIGH);
  // Calculating the distance
  distance = duration * 0.034 / 2;
  // Prints the distance on the Serial Monitor
  //Serial.print("Distance: ");
  Serial.println(distance);
  
  //Serial.write(distance);


    int brightness = Serial.parseInt();
    if(Serial.read() == '\n')
{
      //Serial.println(brightness);
      if(brightness == 1){
        digitalWrite(ledgreen, HIGH);
        digitalWrite(ledyellow, LOW);
        digitalWrite(ledred, LOW);
      }
      else if(brightness == 2){
        digitalWrite(ledyellow, HIGH);
        digitalWrite(ledgreen, LOW);
        digitalWrite(ledred, LOW);
      }
      else if(brightness == 3){
        digitalWrite(ledred, HIGH);
        digitalWrite(ledgreen, LOW);
        digitalWrite(ledyellow, LOW);
      }
    
    
    }
    delay(300);
    //analogWrite(ledgreen, brightness);
  }

Future improvements

I feel like there is a lot that could still be improved. I think the first thing would be to add vertical motion to the car, I was unable to implement it because the ultrasonic sensor detects horizontal distance changes only so it was a bit difficult to manage both the axes. Another factor would be to change the sensor to infrared which would solve the problem of inconsistent readings because infrared sensors send signals in a direct line while ultrasonic sensors send them spread out so the accuracy of readings would be improved. I would also like to add button to the Arduino such as buttons to turn the music on or off and also for the port selection, which I was unable to implement as well.

Concept:

My project is a whack-a-mole game with a rhythm game element where users have to hit the mole at a specific time when a mole’s shadow(a node) aligns with the mole. Users, while the song is playing, must hit the mole whenever the shadow of it

Implementation:

My hardware part of the project is basically a giant system of three switches. Each wire of the circuit connected to its respective digital inputs are attached to an tin foil and left as an uncompleted circuit. The circuit is only closed when the hammer, connected to power, hits the tin foil and completes the circuit, sending a HIGH signal through the digital input. In the UI, the p5 reads a music score and randomly generates nodes related to the notes and rhythm and creates an array. Each node in that array will move towards the designated hit zone, and if the node is aligned with the hit zone, then users can connect the physical circuit to send the signal that the mole was whacked. If the correct mole was whacked, then the user will win points, but if mole was not whacked or the incorrect mole was whacked, then users will lose points. Each song has its specific array of nodes, so the users can achieve a certain max score, and if the user loses more than -1000 points, then the game will be over.

Codes:

-p5js

let musicOne;
let musicTwo;
let musicThree;
let noteImg;
let bg;
let title;
let gameOverSign;
let moleSize = 100;
let shadowImg = [];
let moleImg;
let musicDelay = [4.8, 3.3, 5];
let gameOver = false;
let musicArr = [];
let mole = [];
let colorMole = ["red", "yellow","blue"];
let hit = false;
let score = 0;
let noteArr = [];
let shadowsArr = []
let hitArr = [0, 0, 0];
let firstrun = true;
let startGame = false;
let choice;


function preload(){
  bg = loadImage('bg.png');
  moleImg = loadImage("download.png");
  shadowImg[0] = loadImage("shadowRed.png");
  shadowImg[1] = loadImage("shadowYellow.png");
  shadowImg[2] = loadImage("shadowBlue.png"); 
  shadowImg[3] = loadImage("download.png");
  noteImg = loadImage("note.png");
  gameOverSign = loadImage("gameOver.png")
  title = loadImage("title.png")
  
  musicOne = loadSound("starWars.mp3");
  musicTwo = loadSound("zelda.mp3");
  musicThree = loadSound("canon.mp3");
  
  musicArr.push(musicOne);
  musicArr.push(musicTwo);
  musicArr.push(musicThree);
}

function setup() {
  createCanvas(windowWidth, windowHeight);
  textFont("Comic Sans MS");
  textSize(20);
}

function windowResized() {
  resizeCanvas(windowWidth, windowHeight);
}

function draw() {
  //background image of the game
  //choose a song to play
  background(bg);
  if(!startGame){
    image(title, (windowWidth/2) - windowWidth/16 * 7, (windowHeight/4) - windowHeight/8, windowWidth/8 * 7, windowHeight/3 * 2);
    let stringOne = "Star Wars Theme";
    let stringTwo = "Zelda Theme";
    let stringThree = "Canon";
    fill("white");
    text(stringOne, (windowWidth/4) - (textWidth(stringOne)/2), (windowHeight/18) * 11);
    text(stringTwo, (windowWidth/2) - (textWidth(stringTwo)/2), (windowHeight/18) * 11);
    text(stringThree, (windowWidth/4) * 3 - (textWidth(stringThree)/2), (windowHeight/18) * 11);
    
    image(noteImg, (windowWidth/4) - 55, (windowHeight/10) * 7, 100, 100)
    image(noteImg, (windowWidth/2) - 55, (windowHeight/10) * 7, 100, 100)
    image(noteImg, (windowWidth/4) * 3 - 55, (windowHeight/10) * 7, 100, 100)
  }
  else{
    displayGame();
  }
}

function displayGame(){
  if (!serialActive) {
    text("Press Z to select Serial Port", 20, 30);
  }
  else{
    if(!musicArr[choice].isPlaying() && firstRun){
      musicArr[choice].play(musicDelay[choice]);
      firstRun = false;
    }
    
    if(shadowsArr.length > 0 && score > -1000){
      if(gameOver){
        gameOver = false;
      }
      for(let i = 0; i < 3; i++){
        mole[i].display();
      }
      
      strokeWeight(5);
      
      //setLineDash([5, 10, 30, 10]);
      //line(100, 0, 100, windowHeight);
      text("Score: " + score, windowWidth/5 * 4, windowHeight/40 * 3);
      for(let i = 0; i < shadowsArr.length; i++){
        shadowsArr[i].display();
      }
      //console.log(shadowsArr.length)
    }
    else{
      fill("white");
      musicArr[choice].stop();
      firstRun = false;
      let scoreText = "Score: " + score;
      image(gameOverSign, windowWidth/2 - windowWidth/16 * 5, windowHeight/4 - windowWidth/16 * 2, windowWidth/8 * 5, windowWidth/8 * 3);
      let longText = "Press R to Play Again!";
      //text(shortText, 200 - textWidth(shortText)/2, 100);
      text(longText, windowWidth/2 - textWidth(longText)/2, windowHeight/8 * 7)
      text(scoreText, windowWidth/2 - textWidth(scoreText)/2, windowHeight/4 * 3);
      for(let i = 0; i < shadowsArr.length; i++){
        shadowsArr.splice(i,1);
      }
      gameOver = true;
    }
  }
  
  
  killShadow();

}

function keyPressed(){
  if (keyCode == 90) {
    setUpSerial();
  }
  if (keyCode == 82){
    if(gameOver == true){
      startGame = false;
      score = 0;
    }
  }
  if (keyCode == 70) {
    toggleFullscreen();
  }
}

function readSerial(data) {
  if (data != null) {
    let fromArduino = split(trim(data), ",");
    if (fromArduino.length == 3) {
      for(let i = 0; i < 3; i++){
        hitArr[i] = int(fromArduino[i]);
      }
    }
  }
}


function killShadow(){
  for(let i = 0; i < shadowsArr.length; i++){
    if(shadowsArr[i].x < 40){
      shadowsArr.splice(i, 1);
      score -= 100; 
    }
  }
}

class shadow{
  constructor(x, y){
    this.x = x;
    this.y = (windowHeight/4) * y;
    this.id = y;
    this.hitVal = false;
  }
  
  display(){
    fill("white");
    if(this.x > 50){
      image(shadowImg[this.id - 1], this.x - moleSize/2, this.y - moleSize/2, moleSize, moleSize);
      if(this.x <100){
        this.hitVal = true;
        image(shadowImg[3], this.x - moleSize/2, this.y - moleSize/2, moleSize, moleSize);
      }
      
    }
    this.x -= 2;
  }
}

class Mole{
  constructor(y){
    this.x = 50;
    this.y = (windowHeight/4) * y;
    this.id = y;
  }
  
  search(){
    if(shadowsArr[0].hitVal == true && shadowsArr[0].id == this.id)
      return true;
    else
      return false;
  }
  
  checkHit(){
    if(hitArr[this.id - 1] == 1){
      if(this.search()){
        score += 100;
        hitArr[this.id - 1] = 0;
        shadowsArr.splice(0, 1);
      }
      else{
        score -= 100;
        hitArr[this.id - 1] = 0;
      }
    }
  }
  
  display(){
    if(!gameOver){
      //image(moleImg, this.x - moleSize/2, this.y - moleSize/2, moleSize, moleSize);
      setLineDash[0];
      fill("black")
      ellipse(this.x, this.y + 20, 70, 40)
    }
    this.checkHit();
  }
}

function mouseClicked(){
  if(!startGame){
    for(let i = 1; i < 4; i++){
      mole.push(new Mole(i));
    }
    if(mouseY > (windowHeight/10) * 7 && mouseY < (windowHeight/10) * 7 + 100){
      if(mouseX > (windowWidth/4) - 55 && mouseX < (windowWidth/4) + 55){
        choice = 0;
        loadNotes(choice);
        startGame = true;
      }
      else if(mouseX > (windowWidth/2) - 55 && mouseX < (windowWidth/2) + 55){
        choice = 1;
        loadNotes(choice);
        startGame = true;
      }
      else if(mouseX > (windowWidth/4) * 3 - 55 && mouseX < (windowWidth/4) * 3 + 55){
        choice = 2;
        loadNotes(choice);
        startGame = true;
      }
    }
  }
}

function loadNotes(x){
  let xCo = windowWidth; 
  firstRun = true;
  let datas = [];
  let notes = [];
  let rhythm = [];
  if(x == 0){
    datas = data1;
  }
  if(x == 1){
    datas = data2;
  }
  if(x == 2){
    datas = data3;
  }
  for(let i = 0; i < datas.length; i++){
    if(i % 2 == 0){
      let temp = 0;
      for(let j = 0; j < datas[i].length; j++){
        temp += datas[i].charCodeAt(j);
      }
      temp = (temp % 3) + 1;
      notes.push(temp);
    }
    else{
      if(datas[i] > 0){
        rhythm.push(datas[i] * 20);
      }
      else{
        rhythm.push(datas[i] * 15);
      }
    }
  }
  
  for(let i = 0; i < notes.length; i++){
    shadowsArr.push(new shadow(xCo, notes[i]));
    xCo += rhythm[i];
  }
}

function toggleFullscreen() {
  let fs = fullscreen(); // Get the current state
  fullscreen(!fs); // Flip it!
}

function setLineDash(list) {
  drawingContext.setLineDash(list);
}

-arduino

int lastButton1State = LOW;
int lastButton2State = LOW;
int lastButton3State = LOW;

int buttonState1;
int buttonState2;
int buttonState3;

unsigned long lastDebounceTime1 = 0;
unsigned long lastDebounceTime2 = 0;
unsigned long lastDebounceTime3 = 0;
unsigned long debounceDelay = 20;


void setup() {
  Serial.begin(57600);
  pinMode(10, INPUT);
  pinMode(9, INPUT);
  pinMode(8, INPUT);
}

void loop() {

  int readingOne = digitalRead(10);
  int readingTwo = digitalRead(9);
  int readingThree = digitalRead(8);

  // check to see if you just pressed the button
  // (i.e. the input went from LOW to HIGH), and you've waited long enough
  // since the last press to ignore any noise:

  // If the switch changed, due to noise or pressing:
  if (readingOne != lastButton1State) {
    // reset the debouncing timer
    lastDebounceTime1 = millis();
  }
  if (readingTwo != lastButton2State) {
    // reset the debouncing timer
    lastDebounceTime2 = millis();
  }
  if (readingThree != lastButton3State) {
    // reset the debouncing timer
    lastDebounceTime3 = millis();
  }

  if ((millis() - lastDebounceTime1) > debounceDelay) {
    // whatever the reading is at, it's been there for longer than the debounce
    // delay, so take it as the actual current state:

    // if the button state has changed:
    if (readingOne != buttonState1) {
      buttonState1 = readingOne;

      // only toggle the LED if the new button state is HIGH
      if (buttonState1 == HIGH) {
        Serial.print(readingOne);
        Serial.print(",");
        Serial.print(readingTwo);
        Serial.print(",");
        Serial.println(readingThree);
      }
    }
    
  }

  if ((millis() - lastDebounceTime2) > debounceDelay) {
    // whatever the reading is at, it's been there for longer than the debounce
    // delay, so take it as the actual current state:

    // if the button state has changed:
    if (readingTwo != buttonState2) {
      buttonState2 = readingTwo;

      // only toggle the LED if the new button state is HIGH
      if (buttonState2 == HIGH) {
        Serial.print(readingOne);
        Serial.print(",");
        Serial.print(readingTwo);
        Serial.print(",");
        Serial.println(readingThree);
      }
    }
    
  }

  if ((millis() - lastDebounceTime3) > debounceDelay) {
    // whatever the reading is at, it's been there for longer than the debounce
    // delay, so take it as the actual current state:

    // if the button state has changed:
    if (readingThree != buttonState3) {
      buttonState3 = readingThree;

      // only toggle the LED if the new button state is HIGH
      if (buttonState3 == HIGH) {
        Serial.print(readingOne);
        Serial.print(",");
        Serial.print(readingTwo);
        Serial.print(",");
        Serial.println(readingThree);
      }
    }
    
  }
  lastButton1State = readingOne;
  lastButton2State = readingTwo;
  lastButton3State = readingThree;
}

Communication:

Whenever the user whacks a mole, the arduino, through serial communication, sends an array with three elements. In that array, the element with the array position that correlates to the mole number that was hit will be 1 and the rest that wasn’t hit will be 0. P5, constantly reading for this data, will take the data and update the targetHit array which will contain exactly that same data from arduino. When a shadow note reaches the designated hit zone, the mole instance will automatically check for the targetHit array and if the targetHit array matches the the id of the shadow mole, then the user will score a point. If the targetHit array does not match the id, then the user will lose a point. If the targetHit array was not updated at all, then p5 will recognize that the user didn’t hit any moles and user will lose a point. The communication from p5 to arduino was not necessary for my project, so there is none.

Aspects I am proud of:

Overall, I generally like the idea that physically completing a circuit with a non-traditional switch to make a serial communication since this was an activity we started the arduino chapter with. Also for some unexplainable reason, I really like the idea of whacking something to make something work. I am also very proud of my object orienting programming of the shadow node instance and mole instance interacting with each other to check whether the shadow node is within the designated hit zone and confirming whether the arduino’s sent information matches with the shadow id. That part took some time coming up with the mechanism that worked, but once I did, throughout the project, the mechanism worked like a charm. Lastly, I really like the arrangement of the nodes correlating to the music since I specifically used an algorithm that took the music score and translated the notes to either 1, 2, or 3 by changing the note to its ASCII code and remainder dividing it by 4. Then, I used the length of that note to decide the distance between it and its next node to match the rhythm of the node and the music, and last but not least, I manually tweaked some of the nodes and rhythms so that it specifically matched to some parts of the song. To be honest, this manual peaking took a huge portion of my time spent in this project.

Improvements:

Originally, I wanted to add more to this project by having more hittable nodes, adding another hitting element to use gloves instead of hammers to make the  game more dynamic and interesting, but unfortunately, the technical difficulties of having two hitting objects made it almost impossible to implement, and adding more hittable objects required too much time and resources. The technical difficulty was that the methodology of adding another hitting object was to add another power connected circuit differentiated by different level of resistance, which would then require analog reading instead of digital reading, and this didn’t work out with my existing switch debouncing mechanism and therefore I had to give up on this improvement. However, given more time and information, I would like to add this improvement which would add a whole new spectrum of dynamicness of the game.

Concept:

To summarize, my final project is a game I like to call Treasure Hunt! In this game there is a pile of random items all over the desk, among these random items is one piece of TREASURE. Your goal is to retrieve that treasure. However, there is a radar sensor moving back and forth constantly trying to detect you. The goal is to get this treasure before the radar detects you and before the time runs out! Hence, you got to hide or run away to avoid the radar’s range.

Implementation:

Arduino:

For the arduino aspect of my final project, I used several components:

• A servo motor which would be the radar sensor moving back and forth consistently.
• An ultrasonic sensor which would be mounted on the servo motor and this would be used to detect the distance of the user from the radar sensor.
• A force sensor to detect whether the treasure has been picked up or not. If it was 0 for 1 second, that indicates that the treasure has been found and that you won the game!
• Three arcade LED switches to change the difficulty of the game. Depending on which buttons were pressed, that would change the speed of the servo motor, hence the radar sensor would move much faster and would be harder to avoid the radar’s range in time.
• I also added an LED which is used for debugging and also managing whether the treasure has been set on the force sensor before starting the game. This is because if we started the game and the treasure wasn’t on the force sensor properly, then the game would simply declare the user has won and it wouldn’t make sense.

p5js:

The main aspect of p5js was the map out a visual aid for the radar sensor and to also map out the different stages of the game to Arduino, whether than being the main menu, losing screen, winning screen or the main game screen.

For the main radar screen, it would have the entire mapping of the radar sensor moving back and forth. It also contains details such as whether the object is in range or out of range, the degree of rotation, the distance from the sensor so the user can see how the radar maps them out.

It spins and maps out a green line attempting to detect the user, if the user is detected, it would map out a red line to indicate this and if they are detected for 1 second, then they lose. Else if there were to pick up the treasure then they win!

They can also go back to the menu from this screen to change their difficulty if they wish

Arduino and p5js Interactivity:

I made a little map of how Arduino and p5js map together:

Aspects I’m Proud of:

I would say i’m very proud of how the mapping between the radar sensor through the Arduino and p5js turned out. Visually on the p5js it looks very cool, especially with a really cool blur effect I added on p5js. That way it makes it seem like an actual radar sensor. The mapping between all the different phases of the game are really well done especially through all the values from the Arduino to p5js.

I would say I’m also very proud of how the actual radar sensor box looks like, it looks very cool and the design is very awesome in my opinion. I would also say that the arcade buttons are a very nice touch to display the difficulty of the game. This is because when pressed it lights up indicating which difficulty it is, making it easy for the user to understand what difficulty they have.

Aspects I would Improve:

As per the improvement aspect of the entire project, there are several:

• I would have loved to implement a physical computing aspect for the win/lose stage of the project. My initial ideas were that if you won, then it would activate a servo motor waving a white flag to signify that the radar sensor admits defeat. Also if they lost, then another servo motor would be activated which would unwind a scroll with a picture saying you lost or something. These ideas would’ve been really cool however, because of the time, I was not able to implement these ideas and was forced to simply put a “you lose” or “you win” screen through p5js. However, next time if possible, will definitely implement.
• I would have loved to also improve the quality of the box, perhaps use wood or acrylic or something or stronger material so that the entire product looks much nicer to look at rather than cardboard.
• There is this bug where suppose you lost or won a game, then you would have to wait for the Servo motor to map back to the beginning before you can do anything again. I couldn’t find a solution for this and I think this could also be improved.

Video – Testing on my Own:

Video – User Testing Video:

Video – User Testing #2:

Link to p5js Code:

https://editor.p5js.org/awesomeadi00/sketches/KGKN8f8wE

Arduino Code:

#include <Servo.h>. 

//PINS: 
const int Radar_ServoPin = 10;   

const int echoPin = 9;    
const int trigPin = 8;

const int w_switch = A3;
const int g_switch = A2;
const int y_switch = A1;

const int w_led = 4;
const int g_led = 5;
const int y_led = 6;    

const int pressurePin = A0;
const int pressureLED = 2;

//Global Variables: 
int w_switchVal;
int g_switchVal;
int y_switchVal;
int pressureVal;
unsigned long duration;
int distance;
int difficulty = 1;
int loseBool = 0;

//All game states below are exactly as shown in p5js
int gameState;
int menuState = 0;
int radarState = 1;
int winState = 2;
int loseState = 3;

//Servo object for controlling the servo motor
Servo RadarServo; 

//====================================================================================================================================
void setup() {
  //Begin the serial monitor to p5js at 9600 baud
  Serial.begin(9600);

  //Setting the pinmodes for the ultrasonic sensor
  pinMode(trigPin, OUTPUT);  
  pinMode(echoPin, INPUT);   
  
  //Seting the pinmodes for the arcade switch pins
  pinMode(w_switch, INPUT);
  pinMode(g_switch, INPUT);
  pinMode(y_switch, INPUT);
  
  //Setting the pinmodes for the LEDs in the arcade buttons
  pinMode(w_led, OUTPUT);
  pinMode(g_led, OUTPUT);
  pinMode(y_led, OUTPUT);

  //Output mode for the pressureLED, this will check whether the pressure sensor is active or not.
  pinMode(pressureLED, OUTPUT);

  //Attaching the servos to arduino 
  RadarServo.attach(Radar_ServoPin);  
  RadarServo.write(0);

  //Setup for the game, sets the easy difficulty on by default (difficulty = 1) and sets up the menu screen.
  digitalWrite(w_led, HIGH);
  gameState = 1;

  // Blinks the pressureLED in case the Serial Connection between p5js and Arduino has not been made (debugging)
  while (Serial.available() <= 0) {
    digitalWrite(pressureLED, HIGH);
    Serial.println("0,0"); 
    delay(300);     
    digitalWrite(pressureLED, LOW);
    delay(50);
  }
}

//====================================================================================================================================
void loop() {
  // This checks while the serial connection is available from p5js, then it will parse the gamestate from p5js to arduino. 
  while(Serial.available()) {
    int p5js_gameState = Serial.parseInt();
    if(Serial.read() == '\n') {
      gameState = p5js_gameState;
    }
  }

  //Based on the gamestate from p5js, it will map out exactly what to do on Arduino.
  if(gameState == menuState) {setDifficulty();}
  else if(gameState == radarState) {radarActivate();}
  Serial.println();
}

//====================================================================================================================================
//This functions is only called during the Main Menu. It checks which buttons are pressed to map the speed of the radar, hence setting the difficulty.
void setDifficulty() {
  //If the pressure is greater than 50, then it will switch on the pressureLED to indicate that the user still needs to find it. Else it will switch off indicating they won.
  pressureVal = analogRead(pressurePin);
  if(pressureVal > 200) {digitalWrite(pressureLED, HIGH); }
  else {digitalWrite(pressureLED, LOW); }

  //Reads all the values for all three buttons
  int y_sstate = digitalRead(y_switch);
  int g_sstate = digitalRead(g_switch);
  int w_sstate = digitalRead(w_switch);

  //If the yellow button is pressed, it will switch that on and set hard difficulty to 3
  if(y_sstate == HIGH) {
    digitalWrite(y_led, HIGH);
    digitalWrite(g_led, LOW);
    digitalWrite(w_led, LOW);
    difficulty = 3;
  }

  //If the green button is pressed, it will switch that on and set medium difficulty to 2
  else if(g_sstate == HIGH) {
    digitalWrite(y_led, LOW);
    digitalWrite(g_led, HIGH);
    digitalWrite(w_led, LOW);
    difficulty = 2;           
  }

  //If the white button is pressed, it will switch that on and set easy difficulty to 1
  else if(w_sstate == HIGH) {
    digitalWrite(y_led, LOW);
    digitalWrite(g_led, LOW);
    digitalWrite(w_led, HIGH);
    difficulty = 1;
  }
  Serial.println(difficulty);
}

//====================================================================================================================================
//This function will only be called if the game has started and thus the radar will become active. 
void radarActivate() {
  //Rotates the servo motor from 0 to 180 degrees
  for(int i = 0; i <= 180; i+=difficulty){ 
    pressureVal = analogRead(pressurePin);    //Reads the pressure value to determine if the user has won or not.
    if(pressureVal > 200) {digitalWrite(pressureLED, HIGH); }
    else {digitalWrite(pressureLED, LOW); }
    
    if(gameState == winState || gameState == loseState) {
      RadarServo.write(0); 
      break; 
    }

    RadarServo.write(i);                      
    delay(30);
    distance = calculateDistance();           //Calculates the distance measured by the ultrasonic sensor for each degree. 

    //Sending p5js the degree of rotation of the servo, the distance and the pressure value from the Arduino.
    Serial.print(i+difficulty);              //Sends the current degree to p5js
    Serial.print(",");                  //Sends comma character for indexing in p5js
    Serial.print(distance);             //Sends the distance value to p5js
    Serial.print(",");                  //Sends comma character for indexing in p5js
    Serial.println(pressureVal);          //Sends the pressure value to p5js
  }

  // Repeats the previous lines from 180 to 0 degrees (moving it backwards)
  for(int i = 180; i > 0; i-=difficulty){ 
    pressureVal = analogRead(pressurePin); 
    if(pressureVal > 200) {digitalWrite(pressureLED, HIGH); }
    else {digitalWrite(pressureLED, LOW); }

    if(gameState == winState || gameState == loseState) {
      RadarServo.write(0); 
      break; 
    }

    RadarServo.write(i);
    delay(30);
    distance = calculateDistance();

    Serial.print(i-difficulty);
    Serial.print(",");
    Serial.print(distance);
    Serial.print(",");
    Serial.println(pressureVal);
  }
  Serial.println();
}

//====================================================================================================================================
//Function for calculating the distance measured by the Ultrasonic sensor
//Issue with the sensor is that for objects extremely far away, it takes virtually 69ms to come back and translate to distance
//This is way too long hence, for each increment on the Radar Servo, it would take 69ms.
//Perhaps wrap some board 90cm away from the sensor around in 180˚ so it maps that out and nothing else that's far.
int calculateDistance(){ 
  //Sets the trigPin on LOW for 2 microseceonds.
  digitalWrite(trigPin, LOW); 
  delayMicroseconds(2);
  //Sends a signal from the triggering transducer for 10 microsends.
  digitalWrite(trigPin, HIGH); 
  delayMicroseconds(10);
  digitalWrite(trigPin, LOW);
  
  //Reads the echoPin and returns the sound wave travel time in microseconds. 
  duration = pulseIn(echoPin, HIGH); 

  //The distance will be calculated in cm, hence cm = (microseconds/2)/29
  distance = (duration/2)/29;
  return distance;
}