// VideoTool.cpp : Defines the entry point for the console application.

#include <stdlib.h>
#include <stdio.h>
#include <iostream>
#include <fstream>
#include <string>
#include <tchar.h>
#include <math.h>
#include <time.h>
#include <cassert>
// TODO: reference additional headers your program requires here

#include "cv.h"
#include "highgui.h"

bool alreadyHaveIt(int j,int imageIndices[12]);

// ARGH! why is this necessary ??
#define _NO_HMC_WINSOCK_
#include "SmallSocket.h"
#undef _NO_HMC_WINSOCK_

using namespace std;

// Our Globals!
double Hue_Min = -.1;
double Hue_Max = .5;
double Sat_Min = .12;
double Sat_Max = .99;
double Val_Min = .04;
double Val_Max = .99;
double Val_Dist_Max = 2;
double Val_Dist_Min = .5;

int R_Min = 20;


// these are the two functions you'll need to write
void RGBtoHSV( double r, double g, double b, double *h, double *s, double *v );
void processRed( IplImage* frame, IplImage* f_p , int index);
bool isRed(double h, double s, double v, double r, double g, double b);
bool isGreen(double h, double s, double v, double r, double g, double b);
bool isPurple(double h, double s, double v, double r, double g, double b);
void image_open(IplImage* painted_image, int xlo, int ylo, int xhi, int yhi, int color, uchar threshhold);
void image_close(IplImage* painted_image, int xlo, int ylo, int xhi, int yhi, int color, uchar threshhold);
bool checkforone(IplImage* painted_image, int index, CvPoint top, CvPoint bot);
void analyze(int colorIndex, int index, IplImage* f_p, int xlow, int ylow, int xhi, int yhi);
void textureanal(int colorIndex, int index, IplImage* f_p);
bool isSet(int i, int j, int k);

void setGlobals();  // the code appears at the very bottom of this file

char wndname[] = "VideoTool";
char wndname2[] = "Undistorted image";
char wndname3[] = "Painted image";

char *setwnd[] = {"set0","set1","set2","set3","set4","set5","set6","set7","set8","set9","set10","set11"};

// here, static refers to the fact that these variables
// are local to this file (they're not available in other files,
// which is safer because it allows reuse of their names)
static CvSize imageSize;
static CvPoint old_click_pt;
static CvPoint new_click_pt;
static double framenum;

// global points for drawing boxes, defining loops, etc.
static CvPoint pt1; 
static CvPoint pt2; 
static CvPoint pt3;
static CvPoint pt4;
static CvPoint pt5;
static CvPoint pt6;
static CvPoint pt7;
static CvPoint middleTop;
static CvPoint middleBot;
static CvPoint twoTop;
static CvPoint twoBot;
static CvPoint threeTop;
static CvPoint threeBot;
static CvPoint texTop;
static CvPoint texBot;

// global calibration data
static float* M; 
static float* D;

// info on each of the set cards
static int* number = new int[12];
static int* color = new int[12];
static int* texture = new int[12];
static int* shape = new int[12];
bool displaySets = false;

// more global variables
// these are simply recording all the different states you
// can put the program in via the keyboard
static bool bmpRecording = false;
static bool capturing = true;
static bool getImageFromFile = false;
static bool getImageAgainFromFile = false;
static bool getImage0FromVideoBMPs = false;
static bool saveNextImage = false;
static bool quittingTime = false;
static bool detectRed = false;
static bool showing_savedImage = false;
static bool saveOnlyOneImageToFile = true;
static bool firstTimeForSet = true;

static char* filename = new char[150];
static char* filename2 = new char[150];
static int frameNumber = 0;
static int keyCode = -1;

// pointers to images in memory
static IplImage *frame;
static IplImage* undis_frame;
static IplImage* undis_frame_painted;

static IplImage** setimage_frames;
static IplImage** setimages;
static IplImage** setimages_painted;
static CvArr* undisMap;

// globals for tuning parameters by keyboard...
static int blueValue = 255;

// handle mouse clicks here
void mouse_callback(int event, int x, int y, int flags)
{
	if (event == CV_EVENT_LBUTTONDOWN)
	{
		// cout << "(x,y) = (" << x << "," << y << ")" << endl;

		// reset old_click_pt
		old_click_pt.x = new_click_pt.x;
		old_click_pt.y = new_click_pt.y;

		// get new click point -- note the coordinate change in y
		new_click_pt.x = x;  // coming in from the window system
		new_click_pt.y = imageSize.height-y;  // window system and images have different y axes
	}
}

// print pixel information
void pixelInformation(IplImage* f)
{
	if (f == NULL) return;

	// see if things have changed...
	if (new_click_pt.x != old_click_pt.x || new_click_pt.y != old_click_pt.y) // has it changed?
	{
		if (new_click_pt.x < 0 || new_click_pt.x > 319 || new_click_pt.y < 0 || new_click_pt.y > 239) return;
		// draw a line from old to new
		// cvLine(frame,old_click_pt,new_click_pt,CV_RGB(255,0,0),/*thickness*/1,/*connectivity*/8);
		// set the old = the new
		old_click_pt.x = new_click_pt.x;
		old_click_pt.y = new_click_pt.y;
		// print the RGB values and coordinates of the newly clicked point:
		// note that blue is #0, green is #1, and red is #2 !!

		/* note that this uses the undistorted frame right here... */

		// this is insane!!!
		// see the processRed function for an explanation
		// of why we need "realy"
		int realy = new_click_pt.y;
		if (getImageAgainFromFile || getImageFromFile) 
		{
			//realy = imageSize.height - realy;
		}
		uchar* pixel = cvPtr2D(f,realy,new_click_pt.x);

		cout << "(x,y) = (" << new_click_pt.x << "," << new_click_pt.y << ") with (r,g,b) = ("
			 << (int)(pixel[2]) << "," << (int)(pixel[1]) << "," << (int)(pixel[0]) << ")";

		double hue,sat,val;
		RGBtoHSV((double)pixel[2],(double)pixel[1],(double)pixel[0],&hue,&sat,&val);

		cout << " and (h,s,v) = (" << hue << "," << sat << "," << val << ")" << endl;
	}
}

// the thread for the server
DWORD WINAPI ThreadFunc( LPVOID lpParam ) 
{ 
   	char recvbuf[80];
	char sendbuf[80];

	string t("HI");
	SmallSocket SS(SmallSocket::SERVER);

    sprintf( recvbuf, "Server Started", *(DWORD*)lpParam ); 
    //MessageBox( NULL, szMsg, "ThreadFunc", MB_OK );
	
	while (true)
	{
		SS.receive(recvbuf,80);
		cout << "Received: " << recvbuf << endl;
		
		if (recvbuf[0] == 'q') {
			SS.sendout("q",2);
			// set a variable that tells it to quit...
			quittingTime = true;
		    // cvReleaseCapture( &capture );
			//	cvDestroyWindow("result");
			break;
		}

		if (recvbuf[0] == 'e') {
			// just an example of passing in more information
			// and using sscanf to extract it
			// sscanf(recvbuf,"q %lf %lf %lf ",&savedXforLog,&savedYforLog,&savedTforLog);
			//
			//the connection requires that something come back...
			// note that we're sending two zeros every time here...
			sprintf(sendbuf, "%d %d", 0, 0);
			SS.sendout(sendbuf,80);
		}

		// pause for a bit...
		Sleep(10);
	}

    return 0; 

} 


int main( int argc, char** argv )
{
	// in case you want to log things...
	//ofstream logfile("logfile_out.txt");
	//ifstream loginfile("logfile_in.txt");

	// set up the globals...
	setGlobals();

	// start the server's thread 
	// make sure that all imageprocessing is in main's thread
	DWORD dwThreadId, dwThrdParam = 1; 
    HANDLE hThread; 
    char szMsg[80];

    hThread = CreateThread( 
        NULL,                        // default security attributes 
        0,                           // use default stack size  
        ThreadFunc,                  // thread function 
        &dwThrdParam,                // argument to thread function 
        0,                           // use default creation flags 
        &dwThreadId);                // returns the thread identifier 
 
    // Check the return value for success. 
    if (hThread == NULL) 
    {
        sprintf( szMsg, "CreateThread failed." ); 
        MessageBox( NULL, szMsg, "main", MB_OK );
    }

	/* 
	 * set up the video windows
	 */


	for (int i=0 ; i<12 ; ++i)
		cvNamedWindow( setwnd[i], CV_WINDOW_AUTOSIZE );

	cvNamedWindow( wndname, CV_WINDOW_AUTOSIZE );
	cvNamedWindow( wndname3, CV_WINDOW_AUTOSIZE );
	cvNamedWindow( wndname2, CV_WINDOW_AUTOSIZE );

	/*
	 * set up the mouse input
	 */
	//cvSetMouseCallback( wndname, mouse_callback );
	cvSetMouseCallback( wndname2, mouse_callback );

	/*
	 * more camera-specific stuff
	 */
	CvCapture* capture = 0;
	capture = cvCaptureFromCAM(0);

	// this is allocating the undis_frame once (it will get written each time)
	// get one frame and then copy it...
	if (cvGrabFrame(capture)) 
	{
		frame = cvRetrieveFrame( capture );
		undis_frame = cvCloneImage(frame);
		undis_frame_painted = cvCloneImage(frame);
		// now let's set up the undistortion parameters
		// note that the docs say 3*wider, but that did not work
		// 3*wider _and_ 3*higher worked...
		undisMap = cvCreateMat( imageSize.width*3, imageSize.height*3, CV_32SC3 );
		// need to release this at the end of the program...
		cvUnDistortInit( frame, undisMap, M, D, 1 );
	}
	else
	{
		cout << "No camera??" << endl;
		capturing = false;
	}

	// main loop
	while (!quittingTime)
    {
		if (capturing)
		{
			if( !cvGrabFrame( capture ))   break;
			frame = cvRetrieveFrame( capture );
			if( !frame ) break;

			// cvUnDistortOnce( frame, undis_frame, M, D, 1 );  // might take some extra time...
			cvUnDistort( frame, undis_frame, undisMap, 1 );
			cvReleaseImage(&undis_frame_painted);
			undis_frame_painted = cvCloneImage(undis_frame);
		}
		else if (getImageFromFile || getImageAgainFromFile)
		{
			if (getImageFromFile) {
				// ask for a frame filename
				int i;
				// should factor this path out!!
				if (getImage0FromVideoBMPs) { 
					i = 0;
					getImage0FromVideoBMPs = false;
				}
				else {
					cout << "What image number (in C:\\Documents and Settings\\me\\Desktop\\setVideo\\VideoTool\\VideoBMPs): ";
					cin >> i;
				}
				sprintf(filename2,"C:\\Documents and Settings\\me\\Desktop\\setVideo\\VideoTool\\VideoBMPs\\image%05d.bmp",i);
				// we probably need to release the images returned by cvLoadImage somewhere...
				//frame = cvLoadImage(filename2);
				//undis_frame = cvCloneImage(frame);
				cvReleaseImage(&undis_frame);
				undis_frame = cvLoadImage(filename2);
				cvReleaseImage(&undis_frame_painted);
				undis_frame_painted = cvLoadImage(filename2);
				if (undis_frame == NULL) {
					cout << "Could not find that image.\n";
					continue;
				}
				getImageFromFile = false;
				getImageAgainFromFile = true;
				
			}
			else if (getImageAgainFromFile) {
				cvReleaseImage(&undis_frame_painted);
				undis_frame_painted = cvCloneImage(undis_frame);
			}
		}
		else
		{
			;
		}

		// click only on undis_image
		pixelInformation(undis_frame);//_painted);

		// save the images to a file, if desired
		if (bmpRecording == true && capturing == true)
		{
			sprintf(filename,"C:\\Documents and Settings\\me\\Desktop\\setVideo\\VideoTool\\BMPs\\image%05d.bmp",frameNumber);
			cout << "Captured single image (dist): " << filename <<  endl;
			cvSaveImage(filename,frame);
			sprintf(filename2,"C:\\Documents and Settings\\me\\Desktop\\setVideo\\VideoTool\\BMPsUnd\\image%05d.bmp",frameNumber);
			cout << "Captured single image (undist): " << filename2 <<  endl;
			cvSaveImage(filename2,undis_frame);
			++frameNumber;
			//saveNextImage = false;
		    //if (getImageFromFile) {
			//	bmpRecording = false; 
			//}

			if (saveOnlyOneImageToFile == true)
			{
				bmpRecording = false;
			}
		}

		// here we can add graphics to each frame
		// in order that they show up on the screen
		// we also decide what to display in this routine
		if (detectRed == true)
		{
			processRed(undis_frame, undis_frame_painted, -1);
			for(int i=0; i<12; ++i)
			{
                processRed(setimages[i],setimages_painted[i], i);
				cvShowImage(setwnd[i], setimages_painted[i]);
			}
			//Take every pair and check for third card to match.
			if(displaySets)
				for(int i=0; i<12; i++)
				{
					for(int j=i+1; j<12; j++)
					{
						if(i==j)
							continue;
						for(int k=j+1; k<12; k++)
						{
							if( i==k || j==k )
								continue;
							isSet(i,j,k);
						}
					}
				}
			displaySets = false;
		}

		/*
		 * display everything in the windows
		 */
		cvShowImage(wndname, frame);
		cvShowImage(wndname2, undis_frame_painted);
		cvShowImage(wndname3, undis_frame);

		//handle key presses
		/*
		 * quits
		 */
		if( (keyCode = cvWaitKey( 10 )) == ((int)'q') ) {          
			break;                       // quit everything
		}
		/*
		 * we use = and - to increase and decrease blueValue
		 */
		else if (keyCode == (int)'=') {
			if (blueValue < 251) blueValue += 5;
		}
		else if (keyCode == (int)'-') {
			if (blueValue > 4) blueValue -= 5;
		}
		else if (keyCode == (int)'1') {
			if(Hue_Max - .01 >= -.5) Hue_Max -= .01;
		}
		else if (keyCode == (int)'2') {
			if(Hue_Max + .01 <= 1.0) Hue_Max += .01;
		}
		else if (keyCode == (int)'3') {
			if(Sat_Max - .01 >= 0) Sat_Max -= .01;
		}
		else if (keyCode == (int)'4') {
			if(Sat_Max + .01 <= 1.0) Sat_Max += .01;
		}
		else if (keyCode == (int)'5') {
			if(Val_Max - .01 >= 0) Val_Max -= .01;
		}
		else if (keyCode == (int)'6') {
			if(Val_Max + .01 <= 1.0) Val_Max += .01;
		}
		else if (keyCode == (int)'!') {
			if(Hue_Min - .01 >= -.5) Hue_Min -= .01;
		}
		else if (keyCode == (int)'@') {
			if(Hue_Min + .01 <= 1.0) Hue_Min += .01;
		}
		else if (keyCode == (int)'#') {
			if(Sat_Min - .01 >= 0) Sat_Min -= .01;
		}
		else if (keyCode == (int)'$') {
			if(Sat_Min + .01 <= 1.0) Sat_Min += .01;
		}
		else if (keyCode == (int)'%') {
			if(Val_Min - .01 >= 0) Val_Min -= .01;
		}
		else if (keyCode == (int)'^') {
			if(Val_Min + .01 <= 1.0) Val_Min += .01;
		}
		else if (keyCode == (int)'0'){
			if(Val_Dist_Max +.1 <=3.0) Val_Dist_Max +=.1;
		}
		else if (keyCode == (int)'9'){
			if(Val_Dist_Max -.1 >=0.0) Val_Dist_Max -=.1;
		}
		else if (keyCode == (int)')'){
			if(Val_Dist_Min +.1 <=1.0) Val_Dist_Min +=.1;
		}
		else if (keyCode == (int)'('){
			if(Val_Dist_Min -.1 >=0.0) Val_Dist_Min -=.1;
		}
		else if (keyCode == (int)'7'){
			if(R_Min - 5 >= 0) R_Min -=5;
		}
		else if (keyCode == (int)'8'){
			if(R_Min + 5 <= 255) R_Min +=5;
		}
		else if (keyCode == (int)'p') {
			cout << "Hue_Max: " << Hue_Max << " Hue_Min: " << Hue_Min << endl;
			cout << "Sat_Max: " << Sat_Max << " Sat_Min: " << Sat_Min << endl;
			cout << "Val_Max: " << Val_Max << " Val_Min: " << Val_Min << endl;
			cout << "Val_Dist_Max: " << Val_Dist_Max << "Val_Dist_Min" << Val_Dist_Min << endl;
			cout << "R_Min: " << R_Min << endl;
		}
	    /*
		 * plays the game of Set
		 */
		else if (keyCode == (int)'S') {                           
			capturing = false;
			getImageFromFile = false;
			char filenameset[500]; // for filename...

			/* is this the first play? -- we need to open 12 windows... */
			if (firstTimeForSet == true)
			{
				srand( (unsigned)time( NULL ) );

				cout << "Setting up set!" << endl;
				setimages = new IplImage*[12]; // should delete this somewhere...
				setimages_painted = new IplImage*[12];
				for (int i=0 ; i<12 ; ++i)
				{
				    sprintf(filenameset,"C:\\Documents and Settings\\me\\Desktop\\setVideo\\VideoTool\\VideoBMPs\\image%05d.bmp",i);
					//cvReleaseImage(&undis_frame);
					setimages[i] = cvLoadImage(filenameset); // load image0 12 times.
					setimages_painted[i] = cvLoadImage(filenameset);
					cout << "Setting up window " << i << " named " << setwnd[i] << endl;
					cvShowImage(setwnd[i], setimages[i]);
				}	

				firstTimeForSet = false;
			}
			else {
				cout << "Reading in random images..." << endl;
				int imageIndices[12];
				for (int i=0 ; i<12 ; ++i) { imageIndices[i] = -1; }

				for (int i=0 ; i<12 ; ++i)
				{
					cout << "i is " << i << endl;


					while (1) {
						int randImage = (int)(81.0*rand()/RAND_MAX);
						if (randImage < 0 || randImage > 80) continue;
						cout << "  randImage is " << randImage << endl;
						bool b = alreadyHaveIt(randImage,imageIndices);
						if (!b)
						{
						   imageIndices[i] = randImage;
						}
						else
						{
							cout << "Have it ";
							for (int i=0 ; i<12 ; ++i)
								cout << imageIndices[i] << " ";
							cout << endl;
							continue;
						}
						break;
					}

					sprintf(filenameset,"C:\\Documents and Settings\\me\\Desktop\\setVideo\\VideoTool\\VideoBMPs\\image%05d.bmp",imageIndices[i]);
					//cvReleaseImage(&(setimages[i]));
					setimages[i] = cvLoadImage(filenameset); // load image0 12 times.
					setimages_painted[i] = cvLoadImage(filenameset);
					cout << "Setting up window " << i << " named " << setwnd[i] << " with image #" << imageIndices[i] << endl;
					cvShowImage(setwnd[i], setimages[i]);
				}
			}
			//Find Sets
			detectRed = true;
			displaySets = true;
		} 
		/*
		 * records video to bitmaps (files)
		 */
		else if (keyCode == (int)'f') {                           
			if (bmpRecording) {
				cout << "Stopping the writing to bitmaps\n";
				bmpRecording = false;
			} else {
				cout << "Starting to write to bitmaps\n";
				bmpRecording = true;
			}
			showing_savedImage = false;
		} 
		/*
		 * gets image from a file # in ./VideoBMPs
		 */
		else if (keyCode == (int)'g') {
			cout << "Getting image from file.\n";
			capturing = false;
			getImageFromFile = true;
			getImageAgainFromFile = false;
			bmpRecording = false;
			saveNextImage = false;
			showing_savedImage = true;
		} 
		/*
		 * toggle continuous video
		 */
		else if (keyCode == (int)'s') {
			if (capturing) {
				cout << "Stopping continuous capturing.\n";
				capturing = false;
				getImageFromFile = false;
			}
			else {
				cout << "Starting continuous capturing.\n";
				capturing = true;
				getImageFromFile = false;
				getImageAgainFromFile = false;
				/* reset undis_image */
				// note this assumes that everything went well
				// initially with the first captured frame... !!
				cvReleaseImage(&undis_frame);
				undis_frame = cvCloneImage(frame);
				cvReleaseImage(&undis_frame_painted);
				undis_frame_painted = cvCloneImage(frame);
				// not to mention that these transition images have to be released!
			}
			showing_savedImage = false;
		} 
		else if (keyCode == (int)'e') {
			// emulate server behavior
			saveNextImage = true;
			showing_savedImage = false;
		} 
		/*
		 * toggle finding red (and displaying)
		 */
		else if (keyCode == (int)'r') {
			detectRed = !detectRed;
		}
		/*
		 * the spacebar advances the image in the VideoBMPs directory
		 */
		else if (keyCode == (int)' ') {
			if (showing_savedImage == true)
			{
				int i;
				sscanf(filename2,"C:\\Documents and Settings\\me\\Desktop\\setVideo\\VideoTool\\VideoBMPs\\image%05d.bmp",&i);
				sprintf(filename2,"C:\\Documents and Settings\\me\\Desktop\\setVideo\\VideoTool\\VideoBMPs\\image%05d.bmp",++i);
				cvReleaseImage(&undis_frame);
				undis_frame = cvLoadImage(filename2);
				cvReleaseImage(&undis_frame_painted);
				undis_frame_painted = cvLoadImage(filename2);

				// wrap around!
				if (undis_frame == NULL) 
				{
					i = 0;
					sprintf(filename2,"C:\\Documents and Settings\\me\\Desktop\\setVideo\\VideoTool\\VideoBMPs\\image%05d.bmp",i);
					cvReleaseImage(&undis_frame);
					undis_frame = cvLoadImage(filename2);
					cvReleaseImage(&undis_frame_painted);
					undis_frame_painted = cvLoadImage(filename2);
					
					continue;
				}

				cout << "Getting image " <<  i << " from VideoBMPs.\n";
				getImageAgainFromFile = true;
				getImageFromFile = false;
			}
			/*
			 * initial hit of the space bar...
			 */
			else {
				cout << "No image from file is currently showing..." << endl;
				cout << "Getting image 0 from VideoBMPs.\n";
				capturing = false;
				getImageFromFile = true;
				getImageAgainFromFile = false;
				getImage0FromVideoBMPs = true;
				bmpRecording = false;
				saveNextImage = false;
				showing_savedImage = true;
			}
		}
		
	}

	// release things before quitting
    cvReleaseCapture( &capture );
	cvDestroyWindow("result");

} // end of main

bool alreadyHaveIt(int j,int imageIndices[12])
{
	for (int i=0 ; i<12 ; ++i)
	{
		if (imageIndices[i] == j) return true;
	}
	return false;
}

// just in case you'd like to use these...
inline double mymin(double a, double b) { return a<b?a:b; }

inline double mymax(double a, double b) { return a>b?a:b; }

// read about HSV at http://en.wikipedia.org/wiki/HSV_color_space
// and http://www.cs.rit.edu/~ncs/color/t_convert.html#RGB%20to%20HSV%20&%20HSV%20to%20RGB
//
// feel free to grab code from above or elsewhere online, if you'd like
// be sure you understand what RANGE _your_ code for HSV is "returning"
// in the h, s, and v pointers...
void RGBtoHSV( double r, double g, double b, double *h, double *s, double *v )
{
	double maxRGB;
	double minRGB;
	double delta;

	// get max and min
	if(r >= g && r >= b)
	{
		maxRGB = r;
		if(g >= b)
			minRGB = b;
		else
			minRGB = g;
	}
	else if(g >= b)
	{
		maxRGB = g;
		if(r >= b)
			minRGB = b;
		else
			minRGB = r;
	}
	else
	{
		maxRGB = b;
		if(r >= g)
			minRGB = g;
		else
			minRGB = r;
	}

	delta = maxRGB - minRGB;
	*v = maxRGB/255.0;

	if (maxRGB <= 0.0)
	{
		*h = -1;
		*s = 0;
		return;
	}

	*s = delta/maxRGB;   // better scaling than 255!
	if(delta == 0)
	{
		*h = -1;
		return;
	}

	if(r == maxRGB)
		*h = (g - b)/delta;
	else if (g == maxRGB)
		*h = (1.0/3.0) + (b-r)/delta;
	else
		*h = (2.0/3.0) + (r-g)/delta;
}

// here is where we seek out the red molding and indicate what we've found
// the input named "frame" is the source location of our pixels
// the "output" named f_p is the destination for drawing
void processRed(IplImage* frame, IplImage* f_p, int index)
{
    // some useful local variables...
	int r,g,b;
	double hue,sat,val;

	// pixels are handled as uchar*'s
	uchar* color;     // a pixel from the input, "frame"
	uchar* painted;   // a pixel from the output, "painted"
	uchar* painted2;   // a pixel from the output, "painted"
	uchar* corner;

	// this draws a rectangle from pt4 to pt5 with the color
	// specified by the three RGB values in the CV_RGB macro
	// the R, G, and B channels are all 0 to 255...

	// see the "this is insane!" caution below in order to
	// use graphics on the live video stream...
	cvRectangle(f_p, pt4, pt5, CV_RGB(250,250,0), 1); // 1 == thickness
	//cvLine(f_p, threeTop, threeBot, CV_RGB(250, 250, 250), 1); // line
	//cvCircle( f_p, pt4, 10,  CV_RGB(255,255,0), /*thickness*/ 1 );

	// This loops through the pixels in the rectangle specified
	// by pt4 and pt5 (see setGlobals for their corners)

	int xlow = (int)mymin(pt4.x,pt5.x);
	int xhi = (int)mymax(pt4.x,pt5.x);
	int ylow = (int)mymin(pt4.y,pt5.y);
	int yhi = (int)mymax(pt4.y,pt5.y);

	int matrix[250][250];
	assert(xhi - xlow < 250 && yhi - ylow < 250);
	int colWidth = 10;
	int sampleCols = (xhi-xlow)/colWidth;
	int redTop[25];
	int redBottom[25];
	int redMiddle[25];

	/*for(int i = 0; i < 25; i++)
		redTop[i] = redBottom[i] = -2;*/

	bool foundTop;
	bool foundBottom;
	int temp;
	int temp2;
	int count;
	int i;
	int j;
	int z;
	int y, realy;

	/*int ** matrix;
	matrix= new int*[yhi-ylow];//[xhi-xlow];
	for(int j = 0; j < yhi-ylow; j++)
		matrix[j] = new int[xhi-xlow];*/

	corner = cvPtr2D(f_p, xlow, ylow);

	for (int x = xlow; x<xhi; x++)		//the x direction goes left to right       
	{
		for (int y = ylow; y<yhi; y++)  //the y direction goes top to bottom
		{
			// this is insane!!!
			// unfortunately, access to the pixels is different depending on
			// the source of the pixels: static images and video reverse their
			// y orientations

			// we handle this here...
			realy = y;
			if (getImageAgainFromFile || getImageFromFile) 
			{
				//realy = imageSize.height - realy;  // reverse if static image
			}

			// note that we need to use "realy" in each of these cases
			color = cvPtr2D(frame,realy,x);    // get the pixel we care about
			
			painted = cvPtr2D(f_p,realy,x);    // get the pixel we (might) change

			r = (int)(color[2]);  // the red component of the pixel we care about
			g = (int)(color[1]);  // the green component of the pixel we care about
			b = (int)(color[0]);  // the blue component of the pixel we care about

			// you might want to call this here!
			RGBtoHSV(r,g,b,&hue,&sat,&val);

			// draw on each pixel that we think is "red"
			//if(hue < Hue_Max && hue > Hue_Min && sat < Sat_Max && sat > Sat_Min && val < Val_Max && val > Val_Min)
			// set hue, and make saturation allowable within a range of the value.
			if(isRed(hue,sat,val,r,g,b))///*r > R_Min*/r>g && r>b && hue < Hue_Max && hue > Hue_Min && sat > Sat_Min && sat < val*(Val_Dist_Max) && sat > val*(Val_Dist_Min) && val < Val_Max && val > Val_Min)
			{
				corner[0] = painted[0] = 0;  // sets blue to 0
				corner[1] = painted[1] = 0;    // sets green to 0
				corner[2] = painted[2] = 255;    // sets red to 255
				//cout << "y = " << y << "\t x = " << x << endl;
				//cout << "r = " << y-ylow << "\t x-xlow" << x-xlow << endl;
				matrix[y-ylow][x-xlow]=1;
				//cout << "matrix has: " << matrix[y-ylow][x-xlow] << endl;
			}
			else if(isGreen(hue,sat,val,r,g,b))
			{
				corner[0] = painted[0] = 0;  // sets blue to 0
				corner[1] = painted[1] = 255;    // sets green to 255
				corner[2] = painted[2] = 0;    // sets red to 0
				//cout << "y = " << y << "\t x = " << x << endl;
				//cout << "r = " << y-ylow << "\t x-xlow" << x-xlow << endl;
				matrix[y-ylow][x-xlow]=1;
			}
			else if(isPurple(hue,sat,val,r,g,b))
			{
				corner[0] = painted[0] = 255;  // sets blue to 255
				corner[1] = painted[1] = 0;    // sets green to 0
				corner[2] = painted[2] = 0;    // sets red to 0
				//cout << "y = " << y << "\t x = " << x << endl;
				//cout << "r = " << y-ylow << "\t x-xlow" << x-xlow << endl;
				matrix[y-ylow][x-xlow]=1;
			}
			else
			{
				matrix[y-ylow][x-xlow]=0;
			}

		}
	}
	if(corner[0] == 255) {// blue
		analyze(0, index, f_p, xlow, ylow, xhi, yhi);
	}
	if(corner[1] == 255) {// green
		analyze(1, index, f_p, xlow, ylow, xhi, yhi);
	}
	if(corner[2] == 255) {// red
		analyze(2, index, f_p, xlow, ylow, xhi, yhi);
	}

	if(index < 0 || index > 11)
		return;
	int firstX = xhi;
	int secondX = 0;
	int lastX = 0;
	for(int i=0; i<xhi; i++) //is this at all correct?
	{
		color = cvPtr2D(frame,100,i);    // get the pixel we care about
		r = (int)(color[2]);  // the red component of the pixel we care about
		g = (int)(color[1]);  // the green component of the pixel we care about
		b = (int)(color[0]);  // the blue component of the pixel we care about

		// you might want to call this here!
		RGBtoHSV(r,g,b,&hue,&sat,&val);

		if(isRed(hue,sat,val,r,g,b) || isGreen(hue,sat,val,r,g,b) || isPurple(hue,sat,val,r,g,b))
		{
			secondX=i;
			break;
		}
	}
	for(int i=0; i<xhi; i++) //is this at all correct?
	{
		color = cvPtr2D(frame,80,i);    // get the pixel we care about
		r = (int)(color[2]);  // the red component of the pixel we care about
		g = (int)(color[1]);  // the green component of the pixel we care about
		b = (int)(color[0]);  // the blue component of the pixel we care about

		// you might want to call this here!
		RGBtoHSV(r,g,b,&hue,&sat,&val);

		if(isRed(hue,sat,val,r,g,b) || isGreen(hue,sat,val,r,g,b) || isPurple(hue,sat,val,r,g,b))
		{
			firstX=i;
			break;
		}
	}
	for(int i=0; i<xhi; i++) //is this at all correct?
	{
		color = cvPtr2D(frame,120,i);    // get the pixel we care about
		r = (int)(color[2]);  // the red component of the pixel we care about
		g = (int)(color[1]);  // the green component of the pixel we care about
		b = (int)(color[0]);  // the blue component of the pixel we care about

		// you might want to call this here!
		RGBtoHSV(r,g,b,&hue,&sat,&val);

		if(isRed(hue,sat,val,r,g,b) || isGreen(hue,sat,val,r,g,b) || isPurple(hue,sat,val,r,g,b))
		{
			lastX = i;
			break;
		}
	}

	if(firstX-lastX > 5)									//Diamond
	{
		shape[index] = 1;
	}
	else if( firstX-secondX < -5 || 
			 ( firstX-secondX < 0 && secondX-lastX > 0) )	//Squiggle
	{
		shape[index] = 2;
	}
	else													//Oval
	{
		shape[index] = 0;
	}
}

void setGlobals()
{
	// mouse click locations (old and new, to tell if we've
	// clicked on a new point
	old_click_pt.x = 0;
	old_click_pt.y = 0;
	new_click_pt.x = 0;
	new_click_pt.y = 0;

	// in theory these should get deleted at the end of the program, too...

	// this is the calibration matrix of the camera
	M = new float[9];
	M[0]=    393.1788940f; M[1]=      0.0000000f; M[2]=    162.8373871f;
	M[3]=      0.0000000f; M[4]=    395.7017517f; M[5]=    121.8078156f;
	M[6]=      0.0000000f; M[7]=      0.0000000f; M[8]=      1.0000000f;

	// these are the distortion parameters of the camera
    D = new float[4];
	D[0]=-0.358375f;
	D[1]=0.168058f;
	D[2]=-0.006497f;
	D[3]=0.001283f;

    // theseimage sizes are hard-coded, but should probably be obtained for each
    // image individually -- however, with our cameras and images, this is safe.
	imageSize.width = 320; imageSize.height = 240;

	// set up some global points for drawing, etc.
    int margin = 12;
	pt1.x = margin; pt1.y = margin; 
    pt2.x = imageSize.width-margin; pt2.y = imageSize.height-margin; 

	pt3.x = imageSize.width/2;
	pt3.y = imageSize.height/2;

	pt4.x = 40;//12; 
	pt4.y = 35;//12;

	pt5.x = 280;//300;
	pt5.y = 75;//228;

	middleTop.x = (pt4.x>pt5.x)?(pt4.x+pt5.x)/2:(pt5.x+pt4.x)/2;
	middleTop.y = 35;
	middleBot.x = middleTop.x;
	middleBot.y = 75;
	twoTop.x = 108;
	twoTop.y = 35;
	twoBot.x = 108;
	twoBot.y = 75;
	threeTop.x = 66;
	threeTop.y = 35;
	threeBot.x = 66;
	threeBot.y = 75;

	texTop.y = 119;
	texBot.y = 140;

	pt6.x = 92; 
	pt6.y = 24;

	pt7.x = 228;
	pt7.y = 216;

}

bool isRed(double h, double s, double v, double r, double g, double b)
{
	if(r>(g+b)+10)//15)
		return true;
	return false;
}

bool isGreen(double h, double s, double v, double r, double g, double b)
{
	if(g>(r+b)+15)
		return true;
	return false;
}

bool isPurple(double h, double s, double v, double r, double g, double b)
{
	if(b>(g+r)+3 || r+b+g < 20)
		return true;
	return false;
}

void image_open(IplImage* painted_image, int xlo, int ylo, int xhi, int yhi, int color, uchar threshhold)
{
	uchar* pixel;
	uchar* pixr;
	uchar* pixl;
	uchar* pixu;
	uchar* pixd;
	//xlo++;ylo++; xhi--; yhi--;
	for(int i = xlo; i < xhi; ++i){
		for(int j = ylo; j < yhi; ++j){
			pixel = cvPtr2D(painted_image, j, i);
			pixr = cvPtr2D(painted_image, j-1, i);
			pixl = cvPtr2D(painted_image, j+1, i);
			pixu = cvPtr2D(painted_image, j, i-1);
			pixd = cvPtr2D(painted_image, j, i+1);
			if (pixel[color]>threshhold){
				//cout << "get into the first if statement"<< (int)(pixr[color]) << " " <<(int)(pixl[color]) << " " << (int)(pixu[color]) << " " << (int)(pixd[color]) <<"\n";
				if ((pixr[color]<threshhold && pixr[color]>0) ||
					(pixl[color]<threshhold && pixl[color]>0) ||
					(pixu[color]<threshhold && pixu[color]>0) ||
					(pixd[color]<threshhold && pixd[color]>0)
					){
					//	cout << "got through the if statement\n";
					pixel[color] = 0;
					}
			}
		}
	}
}

void image_close(IplImage* painted_image, int xlo, int ylo, int xhi, int yhi, int color, uchar threshhold)
{
	uchar* pixel;
	uchar* pixr;
	uchar* pixl;
	uchar* pixu;
	uchar* pixd;
	//xlo++;ylo++; xhi--; yhi--;
	for(int i = xlo; i < xhi; ++i){
		for(int j = ylo; j < yhi; ++j){
			pixel = cvPtr2D(painted_image, j, i);
			pixr = cvPtr2D(painted_image, j-1, i);
			pixl = cvPtr2D(painted_image, j+1, i);
			pixu = cvPtr2D(painted_image, j, i-1);
			pixd = cvPtr2D(painted_image, j, i+1);
			if (pixel[color]<threshhold){
				//cout << "get into the first if statement"<< (int)(pixr[color]) << " " <<(int)(pixl[color]) << " " << (int)(pixu[color]) << " " << (int)(pixd[color]) <<"\n";
				if ((pixr[color]>threshhold) ||
					(pixl[color]>threshhold) ||
					(pixu[color]>threshhold) ||
					(pixd[color]>threshhold)
					){
					//	cout << "got through the if statement\n";
					pixel[color] = threshhold;
					}
			}
		}
	}
}

bool checkforone(IplImage* painted_image, int index, CvPoint top, CvPoint bot)
{
	uchar* pixel;
	for(int i = top.y; i < bot.y; ++i){
		pixel = cvPtr2D(painted_image, i, top.x);
		if(pixel[index] == 255)
		{
			cvLine(painted_image, top, bot, CV_RGB(250, 250, 0), 1); // line
			return true;
		}
	}
	return false;
}

void analyze(int colorIndex, int index, IplImage* f_p, int xlow, int ylow, int xhi, int yhi)
{
	if(index != -1) color[index] = colorIndex;
	image_close(f_p, xlow, ylow, xhi, yhi, colorIndex, (uchar)(254)); // clean up image
	image_open(f_p, xlow, ylow, xhi, yhi, colorIndex, (uchar)253);
	cvCircle( f_p, pt4, 10,  CV_RGB((colorIndex == 2?255:0),(colorIndex == 1?255:0),(colorIndex == 0?255:0)), /*thickness*/ 1 );
	if(index != -1) number[index] = 3;
	texBot.x = texTop.x = threeTop.x;
	if(!checkforone(f_p, colorIndex, threeTop, threeBot)) {
		if(index != -1) number[index] = 2;
		texBot.x = texTop.x = twoTop.x;
		if(!checkforone(f_p, colorIndex, twoTop, twoBot)) {
			if(index != -1) number[index] = 1;
			texBot.x = texTop.x = middleTop.x;
			if (!checkforone(f_p, colorIndex, middleTop, middleBot)){
				assert(1==0); //this should never happen
			}
		}
	}

	textureanal(colorIndex, index, f_p);
}

void textureanal(int colorIndex, int index, IplImage* f_p)
{
	int colors = 0;
	int changes = 0;
	double hue, sat, val;
	uchar* current;
	uchar* previous;
	int dr, dg, db;

	//cvLine(f_p, texTop, texBot, CV_RGB(250, 250, 0), 1); // line
	int j = texTop.y; 
	previous = cvPtr2D(f_p, j, texTop.x);
	for(++j; j < texBot.y; ++j){
		current = cvPtr2D(f_p, j, texTop.x);
		RGBtoHSV((double)current[2], (double)current[1], (double)current[0], &hue, &sat, &val);
		if(colorIndex == 0 && isPurple(hue, sat, val, (double)current[2], (double)current[1], (double)current[0]))
			++colors;
		if(colorIndex == 1 && isGreen(hue, sat, val, (double)current[2], (double)current[1], (double)current[0]))
			++colors;
		if(colorIndex == 2 && isRed(hue, sat, val, (double)current[2], (double)current[1], (double)current[0]))
			++colors;
		dr = (current[2]>previous[2])?current[2]-previous[2]:previous[2]-current[2];
		dg = (current[1]>previous[1])?current[1]-previous[1]:previous[1]-current[1];
		db = (current[0]>previous[0])?current[0]-previous[0]:previous[0]-current[0];
		if (dr+dg+db > 75) 
			++changes;
		previous = current;
	}
	if (changes > 5){ //striped pattern
		if (index != -1) texture[index] = 0;
		// print out stripy code!
		CvPoint stripe1, stripe2;
		stripe1.x = texTop.x - 5;
		stripe2.x = texTop.x + 5;
		for (j = texTop.y; j < texBot.y; j += 3) {
			stripe1.y = j;
			stripe2.y = j;
			cvLine(f_p, stripe1, stripe2, CV_RGB(250, 250, 0), 1);
		}
	} else if (colors > 5){ //solid pattern
		if (index != -1) texture[index] = 1;
		cvLine(f_p, texTop, texBot, CV_RGB(250, 250, 0), 10);
	} else { // outline pattern
		if (index != -1) texture[index] = 2;
		texTop.x -= 5;
		texBot.x += 5;
		cvRectangle(f_p, texTop, texBot, CV_RGB(250,250,0), 1); // 1 == thickness
		texTop.x += 5;
		texBot.x -= 5;
	}
}

bool isSet(int i, int j, int k)
{
	//all equal or all distinct
	if( (number[i] == number[j] && number[j] == number[k]) ||
		(number[i] != number[j] && number[j] != number[k] && number[i] != number[k]) )
	{
		if( (color[i] == color[j] && color[j] == color[k]) ||
			(color[i] != color[j] && color[j] != color[k] && color[i] != color[k]) )
		{
			if( (texture[i] == texture[j] && texture[j] == texture[k]) ||
				(texture[i] != texture[j] && texture[j] != texture[k] && texture[i] != texture[k]) )
			{
				if( (shape[i] == shape[j] && shape[j] == shape[k]) ||
					(shape[i] != shape[j] && shape[j] != shape[k] && shape[i] != shape[k]) )
				{
					cout << "Set: " << i << ", " << j << ", " << k << endl;
					return true;
				}
			}
		}
	}
	return false;
}