So cameras, computer vision and cool stuffs! Yay! For this week, I messed around with the Video and OpenCV libraries. this resulted in two programs: 1) a program that taces the webcam capture and runs an edge-identifying algorithm to re-draw the capture in cool, computer-y ways, and 2) an attempt at a function that is able to identify the inside of a shape drawn with a black marker. Fun times.
Program number one was fairly straightforward. I started by opening the OpenCV example that dealt with edge detection to get acquainted with the methods and how to use them. From there, I just hd to slightly modify the sketch: add the Video library and replace the image to be analyzed with the current frame of the camera’s capture. I also added a green tint because that was the first color that came to mind. 😀
Here’s the code (most of the credit goes to the library’s creator):
import processing.video.*;
import gab.opencv.*;
Capture capture;
OpenCV opencv;
PImage src, canny, scharr, sobel;
void setup() {
size(640, 480);
capture = new Capture(this, 640, 480);
capture.start();
}
void draw() {
src = capture;
opencv = new OpenCV(this, src);
opencv.findCannyEdges(30, 85);
canny = opencv.getSnapshot();
opencv.loadImage(src);
opencv.findScharrEdges(OpenCV.HORIZONTAL);
scharr = opencv.getSnapshot();
opencv.loadImage(src);
opencv.findSobelEdges(1, 0);
sobel = opencv.getSnapshot();
pushMatrix();
scale(0.5);
image(src, 0, 0);
tint(0, 255, 0);
image(canny, src.width, 0);
image(scharr, 0, src.height);
image(sobel, src.width, src.height);
popMatrix();
text("Source", 10, 25);
text("Canny", src.width/2 + 10, 25);
text("Scharr", 10, src.height/2 + 25);
text("Sobel", src.width/2 + 10, src.height/2 + 25);
}
void captureEvent(Capture c) {
c.read();
}
And now the painful fun part!
OpenCV comes with a contour-identifying function, which works great, but I couldn’t think of a way to use it to identify only shapes within drawn borders. Instead, it would find many contours that were irrelevant for my purposes.
Thus, I took inspiration from the FindContours example. There, the gray() and threshold() methods are used to turn an image into a grayscale version of itself and then setting all pixels over a certain value of brightness to white, and all others to black. That way, I can work with a simplified version of the image I’m analyzing. (This, however, also introduced some noise, as any shadow dark enough would be turned pitch black, and parts of the outline would be too light to be identified at times).
Then, I wrote I function to look for shapes. It looks at the image pixel by pixel. First, it tries to find an upper left corner. Then it defines that spot as the inside of the shape, and continues marking pixels as inside until it hits an edge, after which it marks the area as outside of the shape. Thus, finding edges while moving through horizontal rows of pixels would trigger the inside/outside of the shape. I also made the program “finish” the shape once it finds a lower right corner.
My algorithm, however, is quite problematic. It can’t deal with (even slightly) concave shapes or shapes whose bottom is thinner than its top. I’ve thought of a couple of ways to address this (noise reduction functions, functions to “smooth” out the analyzed image, making the shape finder function analyze the shape vertically as well as horizontally, etc.), but regardless of the outcome, this was an interesting exercise. 😀
[I’ll insert image here eventually]
And the code. It includes several attempts and upgrades and downgrades (?) and everything. Yay.
import gab.opencv.*;
import processing.video.*;
Capture capture;
OpenCV opencv;
PImage snapshot, analyzed, render;
ArrayList<Contour> contours;
ArrayList<Contour> polygons;
int threshold = 70;
void setup() {
//fullScreen();
size(640, 480);
capture = new Capture(this, 640, 480);
capture.start();
snapshot = capture.get();
opencv = new OpenCV(this, snapshot);
opencv.gray();
opencv.threshold(threshold);
analyzed = opencv.getOutput();
contours = opencv.findContours();
println("found " + contours.size() + " contours");
noFill();
for (Contour contour : contours) {
stroke(0, 255, 0);
contour.draw();
}
render = analyzed;
}
void draw() {
scale(0.5);
image(capture, 0, 0);
//image(snapshot, 640, 0);
image(analyzed, 640, 0);
//image(analyzed, 0, 480);
//for (Contour contour : contours) {
// strokeWeight(3);
// stroke(0, 255, 0);
// pushMatrix();
// translate(640, 0);
// contour.draw();
// stroke(255, 0, 0);
// beginShape();
// for (PVector point : contour.getPolygonApproximation().getPoints()) {
// vertex(point.x, point.y);
// }
// endShape();
// popMatrix();
//}
}
void keyPressed() {
switch (key) {
case ' ':
snapshot = capture.get();
opencv = new OpenCV(this, snapshot);
opencv.gray();
analyzed = opencv.getOutput();
int minVal = 255;
int maxVal = 0;
for (int q = 0; q < analyzed.pixels.length; q ++) {
if (int(red(analyzed.pixels[q])) > minVal) {
minVal = int(red(analyzed.pixels[q]));
}
if (int(red(analyzed.pixels[q])) < maxVal) {
maxVal = int(red(analyzed.pixels[q]));
}
}
threshold = int((minVal + maxVal)*0.3);
opencv.threshold(threshold);
analyzed = opencv.getOutput();
contours = opencv.findContours();
println("found " + contours.size() + " contours");
noFill();
for (Contour contour : contours) {
strokeWeight(3);
stroke(0, 255, 0);
contour.draw();
}
PImage smoothAnalyzed = smoothEdge(analyzed, 5);
detectShape2(smoothAnalyzed, render);
image(smoothAnalyzed, 0, 480);
image(render, 640, 480);
break;
}
}
void captureEvent(Capture c) {
c.read();
}
void detectShape(PImage check, PImage destination) {
boolean shapeFound = false;
boolean inShape = false;
boolean onEdge = false;
boolean checkingCorner = false;
for (int q = 0; q < check.pixels.length; q ++) {
if (red(check.pixels[q]) == 0) { //If black
if (shapeFound == false) { //If you haven't found a shape
checkingCorner = true; //Start checking for corners
} else { //If you have found a shape already
onEdge = true; //That means you are on the edge
}
destination.loadPixels();
destination.pixels[q] = color(255, 255, 255); //All black edges will be white, I only care about inside
destination.updatePixels();
} else { //If white
if (inShape == false && onEdge == true) { //If you were on an edge right before and were outside the shape
onEdge = false; //Then you are not on the edge anymore
inShape = true; //But rather within the shape
} else if (inShape == true && onEdge == true) { //If you were on an edge before and within the shape
onEdge = false; //Then you are ount of the edge now
inShape = false; //And outside of the shape as well
}
if (shapeFound == true && inShape == true) { //If you've found a shape and you are within it
destination.loadPixels();
destination.pixels[q] = color(255, 0, 0); //Mark the pixel
destination.updatePixels();
int corner = 0; //Check if adyacent pixels are black
if (red(check.pixels[q+1]) == 0) { //Check right
corner ++;
}
//if (red(check.pixels[q+1]) == 0) {
// corner ++;
//}
if (q < check.pixels.length - check.width) { //Check down
if (red(check.pixels[q+check.width]) == 0) {
corner ++;
}
}
if (corner >= 2) { //If at least two are black
shapeFound = false; //Then it was a corner and you've found the end of the shape
inShape = false; //therefore you are now outside the shape
}
} else { //Any other white bit that's not in the shape
destination.loadPixels();
destination.pixels[q] = color(255, 255, 255); //Remains white
destination.updatePixels();
}
if (shapeFound == false && checkingCorner == true) { //If you haven't found a shape but are checking for corners
int corner = 0; //Check if adyacent pixels are black
if (red(check.pixels[q-1]) == 0) { //Check left
corner ++;
}
//if (red(check.pixels[q+1]) == 0) {
// corner ++;
//}
if (q >= check.width) {
if (red(check.pixels[q-check.width]) == 0) { //Check up
corner ++;
}
}
boolean lowerEdge = false;
for (int w = floor(q/check.width); w < check.pixels.length/check.width; w ++) {
if (red(check.pixels[q + ((w - floor(q/check.width))*check.width)]) == 0) {
lowerEdge = true;
}
}
if (corner >= 2 && lowerEdge == true) { //If at least two are black
shapeFound = true; //Then it was a corner and you've found a shape
inShape = true; //therefore you are inside the shape
destination.loadPixels();
destination.pixels[q] = color(255, 0, 0); //Mark the pixel!
destination.updatePixels();
} else { //If not, then you still haven't found the inside of the shape
checkingCorner = false; //Stop checking this as a corner
}
}
}
}
}
void detectShape2(PImage check, PImage destination) {
boolean shapeFound = false;
boolean inShape = false;
boolean onEdge = false;
boolean checkingCorner = false;
IntList edgePixels = new IntList();
IntList shapePixels = new IntList();
for (int q = 0; q < check.pixels.length; q ++) {
if (red(check.pixels[q]) == 0) { //If black
if (shapeFound == false) { //If you haven't found a shape
checkingCorner = true; //Start checking for corners
} else { //If you have found a shape already
onEdge = true; //That means you are on the edge
}
edgePixels.append(q); //Add edge pixel to list
} else { //If white
if (inShape == false && onEdge == true) { //If you were on an edge right before and were outside the shape
onEdge = false; //Then you are not on the edge anymore
inShape = true; //But rather within the shape
} else if (inShape == true && onEdge == true) { //If you were on an edge before and within the shape
onEdge = false; //Then you are ount of the edge now
inShape = false; //And outside of the shape as well
}
if (shapeFound == true && inShape == true) { //If you've found a shape and you are within it
shapePixels.append(q); //Add shape pixel to list
int corner = 0; //Check if adyacent pixels are black
if (red(check.pixels[q+1]) == 0) { //Check right
corner ++;
}
//if (red(check.pixels[q+1]) == 0) {
// corner ++;
//}
if (q < check.pixels.length - check.width) { //Check down
if (red(check.pixels[q+check.width]) == 0) {
corner ++;
}
}
if (corner >= 2) { //If at least two are black
shapeFound = false; //Then it was a corner and you've found the end of the shape
inShape = false; //therefore you are now outside the shape
}
} else { //Any other white bit that's not in the shape
destination.loadPixels();
destination.pixels[q] = color(255, 255, 255); //Remains white
destination.updatePixels();
}
if (shapeFound == false && checkingCorner == true) { //If you haven't found a shape but are checking for corners
int corner = 0; //Check if adyacent pixels are black
if (red(check.pixels[q-1]) == 0) { //Check left
corner ++;
}
//if (red(check.pixels[q+1]) == 0) {
// corner ++;
//}
if (q >= check.width) {
if (red(check.pixels[q-check.width]) == 0) { //Check up
corner ++;
}
}
boolean lowerEdge = false;
for (int w = floor(q/check.width); w < check.pixels.length/check.width; w ++) {
if (red(check.pixels[q + ((w - floor(q/check.width))*check.width)]) == 0) {
lowerEdge = true;
}
}
if (corner >= 2 && lowerEdge == true) { //If at least two are black
shapeFound = true; //Then it was a corner and you've found a shape
inShape = true; //therefore you are inside the shape
shapePixels.append(q); //Add shape pixel to the list!
} else { //If not, then you still haven't found the inside of the shape
checkingCorner = false; //Stop checking this as a corner
}
}
}
}
noiseReduction(check, shapePixels, edgePixels);
for (int q = 0; q < destination.pixels.length; q ++) {
destination.loadPixels();
destination.pixels[q] = color(255, 255, 255);
destination.updatePixels();
}
for (int q = 0; q < shapePixels.size(); q ++) {
destination.loadPixels();
destination.pixels[shapePixels.get(q)] = color(255, 0, 0); //Mark the pixel!
destination.updatePixels();
}
for (int q = 0; q < edgePixels.size(); q ++) {
destination.loadPixels();
destination.pixels[edgePixels.get(q)] = color(0, 0, 255);
destination.updatePixels();
}
}
void noiseReduction(PImage check, IntList shape, IntList edge) { //Kill "shapes" that aren't enclosed
boolean edit = false;
int counter = 0;
int initialCount = shape.size();
for (int q = shape.size() - 1; q >= 0; q --) { //For every "shape" pixel...
int analyzedPixel = shape.get(q); //We look at one pixel at a time
if ((analyzedPixel >= check.pixels.length - check.width) //If the pixel belongs to top or bottom row
|| (analyzedPixel <= check.width)) { //By default kill it
shape.remove(q); //BYE
counter ++;
edit = true;
} else { //If it's not on the borders, then the thing gets trickier...
int stay = 0; //Let's count how many "valid" pixels are ther around our buddy
for (int w = 0; w < shape.size(); w ++) { //Valid pixels include those who are also part of the shape
//Check pixel left (get(q)-1), right (get(q)+1), up (get(q)-width), and down (get(q)+width)
//And compare those to all pixels in the shape (get(w))
if (shape.get(q)-1 == shape.get(w) || shape.get(q)+1 == shape.get(w) ||
shape.get(q)-check.width == shape.get(w) || shape.get(q)+check.width == shape.get(w)) {
stay ++; //A match means an adyacent valid pixel
}
}
for (int w = 0; w < edge.size(); w ++) { //As well as those who are part of the edge
//Ditto but now we check if any edge pixel is adyacent
if (shape.get(q)-1 == edge.get(w) || shape.get(q)+1 == edge.get(w) ||
shape.get(q)-check.width == edge.get(w) || shape.get(q)+check.width == edge.get(w)) {
stay ++;
}
}
if (stay < 3) { //Finally, unless the pixel has 4 valid pixels around it...
shape.remove(q); //KILL IT
counter ++;
edit = true;
}
}
}
println(counter + " pixels ignored out of " + initialCount);
if (edit == true) {
//noiseReduction(check, shape, edge);
} else {
return;
}
}
PImage smoothEdge(PImage check) { //To avoid random floating "edge" pixels
IntList edge = new IntList(); //List to store edge pixels
int counter = 0;
for (int w = 0; w < check.pixels.length; w ++) { //Check black pixels for preliminary list
if (red(check.pixels[w]) == 0) {
edge.append(w);
}
}
for (int q = edge.size() - 1; q >= 0; q --) { //For each pixel...
//if ((edge.get(q) >= check.pixels.length - check.width) ||
// (edge.get(q) <= check.width) || (edge.get(q) % check.width == 0) ||
// (edge.get(q) % check.width == check.width - 1)) {
// edge.remove(q);
//}
int nextTo = 0; //Check how many edge pixels are there next to it
for (int e = 0; e < edge.size(); e ++) {
if ((edge.get(e) == edge.get(q) - 1) || (edge.get(e) == edge.get(q) + 1) ||
(edge.get(e) == edge.get(q) - check.width) ||
(edge.get(e) == edge.get(q) + check.width)) {
nextTo ++;
}
}
if (nextTo < 2) { //If there is less than two adyacent edge pixels...
edge.remove(q); //Kill da pixel!
counter ++;
}
}
for (int t = 0; t < check.pixels.length; t ++) { //Now, we blank out the b/w image we had...
check.loadPixels();
check.pixels[t] = color(255, 255, 255);
check.updatePixels();
}
for (int r = 0; r < edge.size(); r ++) { //And write the new black edges on top!
check.loadPixels();
check.pixels[edge.get(r)] = color(0, 0, 0);
check.updatePixels();
}
//for (int q = 0; q < check.pixels.length; q ++) {
// if (q < 20*check.width || q >= check.pixels.length
//}
println(counter + " pixels killed");
return check; //And return the result!
}
PImage smoothEdge(PImage check, int iterations) {
for (int q = 0; q < iterations; q ++) {
check = smoothEdge(check);
}
return check;
}