/**
 * @author Colleen Hamilton
 * @version 1.0
 * 
 * NeuralNetwork.java
 *
 * An implementation of a multi-layer back propogation neural network using 
 * the Mean Squared Error Minimization algorithm.
 */

import java.io.*;			// for buffered streams, getLine etc.
import java.text.*;			// for the DecimalFormat class
import java.util.*;			// for the Random number class and String Tokenizer
import java.lang.Math;		// for activation function utility


public class SpeakerID {

	/**
	 * Class constructor.  Takes a set of DataPoints which determine the
	 * training of the network, and builds a multiple-layer network with
	 * forward propogation 
	 * 
	 * @param numInputs the number of inputs from an outside source that the
	 * network will have
	 * @param layerSpec for each element in the first degree of this array,
	 * a Layer with the activation function at 0 and the number of neurons at
	 * 1 will be created
	 * @param learningRate rate at which to converge upon the correct weights
	 */
	public SpeakerID(int numInputs,
						  int[][] layerSpec,
						  double learningRate) {
		networkLR = learningRate;
		
		networkLayerCount = layerSpec.length;
				
		networkLayers = new Layer[networkLayerCount];
		
		networkNeurons = 0;
		
		competativeNetwork = false;
		
		for(int i = 0; i < networkLayerCount; i++)
			if(layerSpec[i][0] == ActivationFunction.COMPETE)
				competativeNetwork = true;
		
		networkLayers[0] = new Layer(layerSpec[0][0],
									 layerSpec[0][1],
									 numInputs,
									 !competativeNetwork);
		
		networkNeurons += layerSpec[0][1];
		
		for(int i = 1; i < networkLayerCount; i++) {
			networkLayers[i] = new Layer(layerSpec[i][0],
										 layerSpec[i][1],
										 layerSpec[i-1][1],
										 !competativeNetwork);
			
			networkNeurons += layerSpec[i][1];
		}
		
		//print all decimals in the form: "0.000"
		fmt = new DecimalFormat("0.####");
	}
	
   /**
	 * Trains the network to produce the correct output; implementation depends
	 * on the network type
	 * <p>
	 * If the number of expected inputs and outputs does not match the given
	 * numbers, this will fail, and send an error message to the user.
	 * 
	 * @param networkData an array of DataPoints which should be learned
	 * @param batchMode whether or not to batch-train the network
	 * @param iterations specifies the limit on the number of times to iterate
	 * @return none 
	 */
	public void trainNetwork(DataPoint[] networkData, boolean batchMode, int iterations) {
		/**
		 * only move forward if the given input set has the correct number of
		 * inputs and outputs
		 */
		if(inputAcceptable(networkData))
		{
			/**
			 * If the network is not competative, use backpropogation
			 * for training.  Otherwise use competative training to correctly
			 * initialize the weights
			 */
			if(!competativeNetwork) {
				if(!batchMode)
					trainWithBackpropogation(networkData);
				else
					batchTrainWithBackpropogation(networkData);
			}
			else
				competativeTrain(networkData, iterations);
		}
		else {
			System.out.println("Number of inputs or outputs does not match " +
							   "previous given numbers,\ncannot find " +
							   "weights under these conditions.");
		}
	}
	
	/**
	 * Goes through each Layer in the network to find the weights on the inputs
	 * for each, using backpropogation.  Calls upon each layer to find the
	 * weights on its own neurons.
	 * 
	 * @param networkData an array of the training data
	 */
	private void trainWithBackpropogation(DataPoint[] networkData) {
		/**
		 * Now that the training set has been presented, size the arrays
		 * for it
		 */
		networkError = new double[networkData.length]
					[networkLayers[networkLayerCount-1].getNeuronCount()];
	
		for(int L = 0; L < networkLayerCount; L++)
			networkLayers[L].setGradientSize(networkData.length);
	
		boolean errorUnacceptable = true;
		int iterations = 0;
		int LIMIT = 10000;
			
		System.out.println("(iteration,MSE)");
	
		double[] networkInput;
		double[] networkOutput;
					
		do {		
			/**
			 * for each of the given data points, propogate through the
			 * network, training it.
			 */
			for(int d = 0; d < networkData.length; d++) {
				/**
				 * Forward propogate
				 */
				networkInput = networkData[d].getInput();
				
				networkOutput = forwardPropogate(networkInput,true);
				
				/**
				 * Find error
				 */
				findError(networkOutput,networkData[d].getOutput(),d);
				
				/**
				 * Back propogate
				 */
				backPropogate(d);
				
				/**
				 * Modify weights
				 */
				for(int L = networkLayerCount - 1; L >= 0; L--) {
					networkLayers[L].backpropModifyWeights(networkLR,d);				
				}
				
				/**
				 * Test
				 */
				networkOutput = forwardPropogate(networkInput,true);
				
				findError(networkOutput,networkData[d].getOutput(),d);
				
			} //data point loop
			
			/**
			 * Calculate the Mean Squared Error, and compare it to the
			 * acceptable value
			 */
			calculateMSE();
			if(networkMSE < ACCEPTABLE_ERROR)
				errorUnacceptable = false;
			
			if(iterations % 500 == 0)
				System.out.println("("+ iterations +","+ fmt.format(networkMSE)+")");
			
			iterations++;
			
		} while (errorUnacceptable);
		
		System.out.println("Mean Square Error is: " + fmt.format(networkMSE));
		System.out.println("Number of iterations was: " + iterations);
		printNetworkWeights();
	}
	
	/**
	 * Goes through each Layer in the network to find the weights on the inputs
	 * for each, using backpropogation.  Calls upon each layer to find the
	 * weights on its own neurons.
	 * <p>
	 * Trains the network in batch mode; this is where the gradient (the change
	 * in the weights) is averaged over all the data points, instead of for each
	 * individual data point
	 * 
	 * @param networkData the training points for the network
	 */
	private void batchTrainWithBackpropogation(DataPoint[] networkData) {
		/**
		 * Now that the training set has been presented, size the arrays for
		 * it
		 */		
		networkError = new double[networkData.length]
						  [networkLayers[networkLayerCount-1].getNeuronCount()];
		
		for(int L = 0; L < networkLayerCount; L++)
			networkLayers[L].setGradientSize(networkData.length);
	
		double[] networkLastSense =
				new double[networkLayers[networkLayerCount-1].getNeuronCount()];
		
		boolean errorUnacceptable = true;
		int iterations = 0;
		int LIMIT = 500000;

		System.out.println("(iteration,MSE)");
	
		double[] networkInput;
		double[] networkOutput;
						
		do {		
			/**
			 * for each of the given data points, propogate through the
			 * network, training it.
			 */
			for(int d = 0; d < networkData.length; d++) {
				/**
				 * Forward propogate
				 */
				networkInput = networkData[d].getInput();
			
				networkOutput = forwardPropogate(networkInput,true);
					
				/**
				 * Find error
				 */
				findError(networkOutput,networkData[d].getOutput(),d);
				
				/**
				 * Back propogate
				 */
				backPropogate(d);
									
			} //data points
				
			/**
			 * Set the each network gradient to its average
			 * over all data points
			 */
			for(int L = 0; L < networkLayerCount; L++) {
				for(int n = 0; n < networkLayers[L].getNeuronCount(); n++) {
					for(int w = 0; 
						w < (networkLayers[L].getWeights()[n].length); w++) {
						//new gradient value
						double newGrad = 0;
						for(int d = 0; d < networkData.length; d++) {
							newGrad += networkLayers[L].getGradient(d,n,w);
						
						}
						newGrad = newGrad / networkData.length;
						networkLayers[L].setGradient(0,n,w,newGrad);
					}
				}
			}
			
			networkLastSense =
					networkLayers[networkLayerCount-1].getSensitivities();
			
			/**
			 * Modify weights
			 */
			for(int L = networkLayerCount - 1; L >= 0; L--) {
				networkLayers[L].backpropModifyWeights(networkLR,0);
			}
				
			/**
			 * Test
			 */
			for(int d = 0; d < networkData.length; d++) {
				networkInput = networkData[d].getInput();
				
				networkOutput = forwardPropogate(networkInput,true);
				
				findError(networkOutput,networkData[d].getOutput(),d);
				
			} //data points
			
			/**
			 * Calculate the Mean Squared Error, and compare it to the
			 * acceptable value
			 */
			calculateMSE();
			if(networkMSE < ACCEPTABLE_ERROR)
				errorUnacceptable = false;
			
			
			if(iterations % 500 == 0)
				System.out.println("("+ iterations +","+ fmt.format(networkMSE)+")");
			
			iterations++;
			
		} while (errorUnacceptable);
		
		System.out.println("Mean Square Error is: " + fmt.format(networkMSE));
		System.out.println("Number of iterations was: " + iterations);
		printNetworkWeights();
	}
	
	/**
	 * Sets the weights of the network to be prototype vectors of the training
	 * data for pattern recognition.
	 * 
	 * @param networkData the data points to train on
	 * @param iterations the number of times to look at each data point
	 */
	private void competativeTrain(DataPoint[] networkData, int iterations) {
		/**
		 * Check if this is a Kohonen or LVQ2 network:
		 *		layer count = 1: Kohonen
		 *		layer count = 2: LVQ2
		 */
		if(networkLayerCount == 2) {
			LVQ2Train(networkData, iterations);
		}
		else {
			System.out.println("This is an unsupported network type, which will " +
							   "not train.");
		}
	}
	
	/**
	 * Uses the LVQ2 method to train the network
	 * 
	 * @param networkData the data points to train on
	 * @param iterations the number of times to look at each data point
	 */
	private void LVQ2Train(DataPoint[] networkData, int iterations) {
		/**
		 * Now that the training set has been presented, size
		 * the arrays for it
		 */		
		networkError = new double[networkData.length]
						  [networkLayers[networkLayerCount-1].getNeuronCount()];
		
		/**
		 * Set up the weight vector for layer 2
		 */
		int L0count = networkLayers[0].getNeuronCount();
		int L1count = networkLayers[1].getNeuronCount();
		
		// Create a matrix of the format:
		// {{1,1,0,0,0,0,0,0},
		//  {0,0,1,1,0,0,0,0},
		//  {0,0,0,0,1,1,0,0},
		//  {0,0,0,0,0,0,1,1}};
		double[][] weightVector = new double[L1count][L0count];
		int k = 0;
		for(int i = 0; i < L1count; i++) {
			for(int j = 0; j < L0count; j++)
				weightVector[i][j] = 0;
			for(int m = 0; m < 2; m++) {
				weightVector[i][k] = 1;
				k++;
			}
		}
		
		networkLayers[1].initializeWeights(false);
		networkLayers[1].setWeights(weightVector);
		
		int numPoints = networkData.length;
		
		boolean toward = true;
		boolean away = false;
		
		double[] networkInput;
		double[] networkOutput;
		
		for(int i = 0; i < iterations; i++) {
		for(int d = 0; d < numPoints; d++) {
			
			/**
			 * forward propogate
			 */
//			System.out.println("First Run:");
			networkInput = networkData[d].getInput();
			networkOutput = forwardPropogate(networkInput,true);
			
			/**
			 * find out which weights need changing
			 */
			findError(networkOutput,networkData[d].getOutput(),d);
			int L1to = -1;
			boolean outputError = false;
			int L0to = -1;
			int L0away = -1;
			
//			System.out.print("Layer error : ");
//			printMatrix(networkError[d],true);
			
			//if the error vector contains a -1, that neuron needs to move away
			//if the error vector contains a 1, that neuron needs to move closer
			for(int e = 0; e < networkError[d].length; e++) {
				if(networkError[d][e] == -1)
					outputError = true;
				if(networkError[d][e] == 1) {
					L1to = e;
					outputError = true;
				}
			}
			
			//get the L0 output vector (the result of the competition), to use
			//the winning neuron
			double[] L1input = networkLayers[0].getOutputs();
			for(int e = 0; e < L1input.length; e++)
				if(L1input[e] == 1) {
					if(!outputError)
						//no error found, move the winning L0 neuron closer
						L0to = e;
					else
						//there was an error found, move the winning L0 neuron away
						L0away = e;
				}
			
			//FIXME
			int[] acceptableNeurons = new int[2];
			
			//if there was an output error, find which neuron must
			//come closer using L1to
			if(outputError) {
				
				int a = 0;
				for(int w = 0; w < weightVector[L1to].length; w++) {
					if(weightVector[L1to][w] == 1) {
						acceptableNeurons[a] = w;
						a++;
					}
				}
				L0to = findClosestAcceptableNeuron(networkInput,acceptableNeurons);
			}	
			/**
			 * modify weights
			 */
			networkLayers[0].competativeModifyWeights(networkLR, L0to, toward);
			if(outputError)
				networkLayers[0].competativeModifyWeights(networkLR, L0away, away);
			
			/**
			 * Test
			 */
//			System.out.println("Improved run:");
			networkOutput = forwardPropogate(networkInput,true);
//			System.out.print("Layer error : ");
			findError(networkOutput,networkData[d].getOutput(),d);
//			printMatrix(networkError[d],true);
			
		}
		}
	}
	
	/**
	 * Runs a forward-propogation and error test of the given data, to show
	 * how well the network has been trained.
	 * <p>
	 * If the number of expected inputs and outputs does not match the given
	 * numbers, this will fail, and send an error message to the user.
	 * 
	 * @param networkData the points to be checked
	 * @return none
	 */
	public void testNetwork(DataPoint[] networkData) {
		networkError = 
				new double[networkData.length]
						  [networkLayers[networkLayerCount-1].getNeuronCount()];
		
		if(inputAcceptable(networkData))
		{
			/**
			 * Test the network
			 */
			double[] networkInput;
			double[] networkOutput;
			
			int errorCount = 0;
			for(int d = 0; d < networkData.length; d++) {
				boolean error = false;
				/**
				 * Forward propogate
				 */
//				System.out.println("Testing point " + (d+1) + "...");
				networkInput = networkData[d].getInput();
				networkOutput = forwardPropogate(networkInput,false);
				int out = 0;
				while(networkOutput[out] != 1)
					out++;
//				System.out.print("Output is: ");
//				printMatrix(networkOutput,true);
				
				/**
				 * Find error
				 */
				findError(networkOutput,networkData[d].getOutput(),d);
//				System.out.print("Error is: ");
//				printMatrix(networkError[d],true);
				int err = 0;
				while(networkData[d].getOutput()[err] != 1)
					err++;
				
				for(int i = 0; i < networkError[d].length; i++) {
					if(networkError[d][i] == -1) {
						errorCount++;
						System.out.println("File " + networkData[d].getFile() + " was classified " +
										   "incorrectly: was " + i + ", should " +
										   "have been " + err + ".");
					}
				}
				
			}
			
			System.out.println(networkData.length + " files tested;");
			System.out.println(errorCount + " files incorrectly categorized:");
			System.out.println((networkData.length-errorCount) +
							   "/" + networkData.length);
		}
		else {
			System.out.println("Number of inputs or outputs does not match " +
							   "previous given numbers,\ncannot find " +
							   "weights under these conditions.");
		}
	}
	
	/**
	 * Pushes the given input through to the end of the network, generating
	 * some kind of output, which can then be tested.
	 * 
	 * @param networkInput the array which constitutes the input for the first layer
	 * @param training a flag to indicate if forward propogation is being used to train
	 * @return the result of the forward propogation
	 */
	public double[] forwardPropogate(double[] networkInput, 
									  boolean training) {
		for(int L = 0; L < networkLayerCount; L++) {
//			System.out.print("Input for layer " + (L+1) + " is: ");
//			printMatrix(networkInput,true);
			networkInput = networkLayers[L].forwardPropogate(networkInput, 
															 training,
															 competativeNetwork);
//			System.out.print("Output for layer " + (L+1) + " is: ");
//			printMatrix(networkInput,true);
		}
		return networkInput;
	}
	
	/**
	 * Finds the error (desired output - actual output) for the network's
	 * current output.
	 * 
	 * @param actual the network's current output
	 * @param desired the network's goal output
	 * @param dataPoint the number of the point that the error is being found for
	 * @return none
	 */
	public void findError(double[] actual, double[] desired, int dataPoint) {
		int desiredLength = desired.length;
		
		for(int i = 0; i < desiredLength; i++) {
			networkError[dataPoint][i] = desired[i] - actual[i];
		}
	}
	
	/**
	 * Calculates the sensitivities of each neuron in each layer beginning
	 * with the last layer, and back propogating to the first.
	 * 
	 * @param d the current data point that the error has been found for
	 * @return none
	 */
	public void backPropogate(int d) {
		double[] networkSensitivity = 
					networkLayers[networkLayerCount - 1].
										findSensitivityM(networkError[d],d);
		
		for(int L = networkLayerCount - 2; L >= 0; L--) {
			networkSensitivity = 
					networkLayers[L].backPropogate(networkSensitivity,
											networkLayers[L+1].getWeights(),
											d);
		}
	}
	
	/**
	 * Given a list of acceptable neurons (neurons which make up
	 * a subcategory in a classification), find the one which has
	 * the closest output
	 * 
	 * @param networkInput the data which is the input for the neurons
	 * @param acceptableNeurons the neurons which may participate in the
	 * competition
	 * @return the number of the neuron that won
	 */
	public int findClosestAcceptableNeuron(double[] networkInput,
										   int[] acceptableNeurons) {
		
		double[] output = networkLayers[0].limitForwardProp(networkInput,
															acceptableNeurons);
		
		int returnable = -1;
		for(int i = 0; i < output.length; i++)
			if(output[i] == 1)
				returnable = i;
		return returnable;
	}
	
	/**
	 * Calculates the mean square error of the network
	 * 
	 * @return none
	 */
	public void calculateMSE() {
		networkMSE = 0;
		for(int e = 0; e < networkError[0].length; e++) {
			for(int d = 0; d < networkError.length; d++)
				networkMSE += (networkError[d][e] * networkError[d][e]);
			networkMSE = networkMSE/networkError.length;
		}
	}
	
	/**
	 * Verifies if the given data set is of the same proportions as the
	 * network
	 * 
	 * @param the input data set
	 * @return true if the data set is the same size as the network
	 */
	public boolean inputAcceptable(DataPoint[] points) {
		int numInputsAllowed;
		int numOutputsAllowed;
		
		if(!competativeNetwork)
			numInputsAllowed = networkLayers[0].getWeights()[0].length-1;
		else
			numInputsAllowed = networkLayers[0].getWeights()[0].length;
		
		numOutputsAllowed = networkLayers[networkLayerCount-1].getNeuronCount();
		
		if((points[0].getInput().length == numInputsAllowed) &&
		   (points[0].getOutput().length == numOutputsAllowed))
			return true;
		else
			return false;
	}
	
	/**
	 * Prints the weights of the network as it currently stands
	 * 
	 * @return none
	 */
	public void printNetworkWeights() {
		for(int L = 0; L < networkLayerCount; L++) {
			System.out.println("Layer " + (L+1));
			for(int n = 0; n < networkLayers[L].getNeuronCount(); n++) {
				System.out.println("\tneuron " + (n+1));
				for(int w = 0; 
					w < networkLayers[L].getWeights()[n].length; 
					w++) {
					double recievedWeights[][] = networkLayers[L].getWeights();
					if(w == 0)
						System.out.println("\t\tbias is: " +
										   fmt.format(recievedWeights[n][w]));
					else	
						System.out.println("\t\tweight " + w + " is: " +
										   fmt.format(recievedWeights[n][w]));
				}
			}
		}
	}
	
	/**
	 * Prints out the given array with brackets and commas, as if it were a
	 * matrix.
	 * 
	 * @param input the double array to be printed
	 * @param addNewLine a flag as to whether to end with a \n character
	 * @return none
	 */
	public void printMatrix(double[] input, boolean addNewLine) {
		int inputLen = input.length - 1;
		
		System.out.print("[");
		for(int j = 0; j <= inputLen - 1; j++)
			System.out.print(fmt.format(input[j]) + ", ");
		System.out.print(fmt.format(input[inputLen]) + "]");
		
		if(addNewLine)
			System.out.println();
	}
	
	/**
	 * Prints out the given array with brackets and commas, as if it were a
	 * matrix.
	 * 
	 * @param input the integer array to be printed
	 * @param addNewLine a flag as to whether to end with a \n character
	 * @return none
	 */
	public void printMatrix(int[] input, boolean addNewLine) {
		int inputLen = input.length - 1;
		
		System.out.print("[");
		for(int j = 0; j <= inputLen - 1; j++)
			System.out.print(fmt.format(input[j]) + ", ");
		System.out.print(fmt.format(input[inputLen]) + "]");
		
		if(addNewLine)
			System.out.println();
	}
		
	/**
	 * A constant multiplier which factors into each adjustment made to
	 * network weights during changes
	 */
	private double networkLR;
	
	// A formatter, for pretty-ing up all those ugly long decimals
	private DecimalFormat fmt;
	
	// The Mean Squared Error of the network
	private double networkMSE;
	// The total number of neurons in this network
	private int networkNeurons;
	// The number of layers in this network
	private int networkLayerCount;
	// The error of the network for each data point 
	private double[][] networkError;
	// The layers of this network
	private Layer[] networkLayers;
	// Whether or not this network has a competative layer
	private boolean competativeNetwork;
	
	//an acceptable point for "zero" sensitivity
	private static final double ACCEPTABLE_ERROR = 0.01;
	
			
public class Layer {
		
	/**
	 * Class constructor.
	 * 
	 * @param layerFn the activation function to use in all neurons in
	 * this layer
	 * @param numNeurons the number of neurons in this layer
	 * @param numInputs the number of inputs to this layer
	 * @param useBias true if this network should use a bias weight
	 */
	public Layer(int layerFn, int numNeurons, int numInputs, boolean useBias) {
		layerFunction = layerFn;
		
		/**
		 * Pick the appropriate training function
		 */
		if((layerFunction == ActivationFunction.HARDLIM) || 
		   (layerFunction == ActivationFunction.SATLIN))
			layerTrainingFunction = ActivationFunction.LOGSIG;
		else if((layerFunction == ActivationFunction.HARDLIMS) || 
				(layerFunction == ActivationFunction.SATLINS))
			layerTrainingFunction = ActivationFunction.TANSIG;
		else
			layerTrainingFunction = layerFunction;
		
		layerNeuronCount = numNeurons;
		
		//tack on an extra weight for the bias
		if(useBias)
			layerWeights = new double[layerNeuronCount][numInputs+1];
		else
			layerWeights = new double[layerNeuronCount][numInputs];
		
		layerInput = new double[numInputs];
		layerOutput = new double[layerNeuronCount];
		
		randomGen = new Random();
		
		initializeWeights(true);
	}
	
	/**
	 * Initializes the weights of this layer using either random weights
	 * or 0 weights
	 * 
	 * @param useRandom use random doubles to initialize the weights;
	 * otherwise 0 is used
	 */
	public void initializeWeights(boolean useRandom) {

		for(int i = 0; i < layerWeights.length; i++)
			for(int j = 0; j < layerWeights[i].length; j++) {
				if(useRandom)
					layerWeights[i][j] = randomdouble();
				else
					layerWeights[i][j] = 0;
			}
	}
	
	/**
	 * Pushes the given input through this layer, generating output.
	 * 
	 * @param layerData an array of inputs which should be learned
	 * @param training a flag which indicates which layer function to use
	 * @param compete a flag which, if true, means there is no bias weight,
	 * because this is a competative network
	 * @return output of this layer
	 * @see NeuralNetwork#trainNetwork
	 * @see NeuralNetwork#batchTrainNetwork
	 * @see NeuralNetwork#testNetwork
	 */
	public double[] forwardPropogate(double[] layerData, boolean training, boolean compete) {
		layerInput = layerData;
		
		//find weights for each neuron in the layer
		
		int fn;
		if(training)
			fn = layerTrainingFunction;
		else
			fn = layerFunction;
		
		int inputLength = layerInput.length;
		for(int n = 0; n < layerNeuronCount; n++) {
			// Calculate actual output for this input point
			double actual = 0;
			for(int i = 0; i < inputLength; i++)
				actual += layerWeights[n][i]*layerInput[i];
			
			if(!compete)
				// add the bias
				actual += layerWeights[n][inputLength];
			
			actual = ActivationFunction.function(fn,actual);
			layerOutput[n] = actual;
			
		}//end neurons in the layer loop
		
		if(fn == ActivationFunction.COMPETE) {
			ActivationFunction.competeFunction(layerOutput);
		}	
		
		return layerOutput;
	}
	
	/**
	 * Assumes that this is the last layer in the network, and calculates
	 * the sensitivity accordingly.
	 * 
	 * @param error the current error matrix of the network as a whole
	 * @param d the data point that this is a sensitivity for
	 * @return the sensitivity matrix for this layer
	 * @see NeuralNetwork#trainNetwork
	 * @see NeuralNetwork#batchTrainNetwork
	 * @see NeuralNetwork#testNetwork
	 */
	public double[] findSensitivityM(double[] error, int d) {
		layerSensitivities = new double[layerNeuronCount];
					
		for(int i = 0; i < layerNeuronCount; i++) {
			layerSensitivities[i] = -2*
					ActivationFunction.derivative(layerTrainingFunction,
												  layerOutput[i])*
							 error[i];
		}
		
		calculateGradients(d);
		
		return layerSensitivities;
	}
	
	/**
	 * Backward propogates through the network, changing weights so that
	 * correct output is reached.
	 * 
	 * @param sOld the sensitivity of the next layer in the network
	 * @param nextWeights the weights of the next layer in the network
	 * @param d the data point that is being back propogated for
	 * @return sensitivity of this layer
	 * @see NeuralNetwork#trainNetwork
	 * @see NeuralNetwork#batchTrainNetwork
	 * @see NeuralNetwork#testNetwork
	 */
	public double[] backPropogate(double[] sOld,double[][] nextWeights,int d) {
		
		layerSensitivities = new double[layerNeuronCount];
		
//		System.out.print("Old sensitivity is: ");
//		printMatrix(sOld,true);
		
		for(int n = 0; n < layerNeuronCount; n++) {
			layerSensitivities[n] = 0;
			
			//s = next layer's neuron
			for(int s = 0; s < sOld.length; s++) {
				for(int w = 1; w < nextWeights[s].length; w++) {
					layerSensitivities[n] += nextWeights[s][w]*sOld[s];
/*					System.out.println("layerSensitivity just added " +
									   "nextWeight[" + s + "][" + w + "]*sOld["
									   + s + "]: " + nextWeights[s][w] + "*" +
									   sOld[s] + " = " + 
									   (nextWeights[s][w]*sOld[s]));
*/				}
			}
			
/*			System.out.print("layerSensitivity (" + layerSensitivities[n] +
							   ") will be multiplied by " + 
							   ActivationFunction.derivative(layerTrainingFunction,
														   layerOutput[n]) + " to become: ");
*/			layerSensitivities[n] *= 
							ActivationFunction.derivative(layerTrainingFunction,
												layerOutput[n]);
//			System.out.println(layerSensitivities[n]);
			
		}
		
		calculateGradients(d);
		
		return layerSensitivities;
	}
	
	/**
	 * Forward propogates through the layer using only the specified neurons;
	 * this is useful in competative networks.  This method assumes it has been
	 * called by a competative network.
	 * 
	 * @param layerData the input to the layer
	 * @param acceptableNeurons the neurons which are to be used in this forward
	 * propogation
	 */
	public double[] limitForwardProp(double[] layerData, int[] acceptableNeurons) {
		
		double[] limitedOutput = new double[layerNeuronCount];
		//initialize limitedOutput array to minimum possible output
		for(int i = 0; i < layerNeuronCount; i++) {
			limitedOutput[i] = Integer.MIN_VALUE;
		}
		
		for(int i = 0; i < acceptableNeurons.length; i++) {
			int n = acceptableNeurons[i];
			
			// Calculate actual output for this input point
			double actual = 0;
			int inputLength = layerData.length;
			for(int j = 0; j < inputLength; j++)
				actual += layerWeights[n][j]*layerData[j];
			
			limitedOutput[n] = actual;
		}
		
		limitedOutput = ActivationFunction.competeFunction(limitedOutput);	
		
		return limitedOutput;
	}
	
	/**
	 * Sets up the size of the gradient array using the number of data points
	 * 
	 * @param d the number of data points to be gradient-ed over
	 * @return none
	 */
	public void setGradientSize(int d) {
		layerGradients = new double[d][layerNeuronCount]
							[layerWeights[0].length];
	}
	
	/**
	 * Calculates the gradient array for this layer, which is what the weight
	 * matrix modifies itself by
	 * 
	 * @param d the data point that these gradients are for
	 * @return none
	 */
	public void calculateGradients(int d) {
		for(int n = 0; n < layerNeuronCount; n++) {
			int weightsLength = layerWeights[n].length;
			for(int w = 0; w < weightsLength-1; w++) {
/*				System.out.println("layer gradient for point " +
								   d + ", neuron " + n + ", weight " +
								   w + ", is: " + (layerSensitivities[n]*
												   layerInput[w]));
*/				layerGradients[d][n][w] = layerSensitivities[n]*layerInput[w];
			}
			//bias weight
/*			System.out.println("layer gradient for point " +
							   d + ", neuron " + n + ", bias weight " +
							   "is: " + layerSensitivities[n]);
*/			layerGradients[d][n][weightsLength-1] = layerSensitivities[n];
		}
	}
	
	/**
	 * Modifies the weights of this layer based on the
	 * backpropogation weight change method
	 * 
	 * @param LR the learning rate
	 * @param d the data point of this weight modification
	 * @return none
	 * @see NeuralNetwork#trainNetwork
	 * @see NeuralNetwork#batchTrainNetwork
	 */
	public void backpropModifyWeights(double LR, int d) {
		for(int n = 0; n < layerWeights.length; n++)
			for(int w = 0; w < layerWeights[n].length; w++) {
				layerWeights[n][w] = layerWeights[n][w] - 
									 (LR*layerGradients[d][n][w]);
			}
	}
	
	/**
	 * Modifies the weights of this layer based on the LVQ2
	 * weight change method
	 * 
	 * @param LR the learning rate
	 * @param neuron the neuron who's weight must change
	 * @param toward a flag which if true, means the change is additive
	 * (subtractive if false)
	 * @return none
	 * @see NeuralNetwork#trainNetwork
	 * @see NeuralNetwork#batchTrainNetwork
	 */
	public void competativeModifyWeights(double LR, int neuron, boolean toward) {
		int i = 1;
		if(!toward)
			i = -1;
		
		for(int w = 0; w < layerWeights[neuron].length; w++) {
			layerWeights[neuron][w] = layerWeights[neuron][w] +
								 i*LR*(layerInput[w]-layerWeights[neuron][w]);
		}
	}
	
	/**
	 * Gets the number of neurons in this layer
	 */
	public int getNeuronCount() {
		return layerNeuronCount;
	}
	
	/**
	 * Returns the outputs of this layer.
	 * 
	 * @return an array containing the output of each neuron in this layer
	 */
	public double[] getOutputs() {
		return layerOutput;
	}
	
	/**
	 * Returns the weights of this layer.
	 * 
	 * @return a 2D array, where the first dimension is a neuron and the
	 * second dimension is the weights on the inputs of that neuron
	 */
	public double[][] getWeights() {
		return layerWeights;
	}
	
	/**
	 * Returns the activation function for this layer
	 * 
	 * @return one of the possible activation functions from the
	 * ActivationFunction class
	 */
	public int getActivationFunction() {
		return layerFunction;
	}
	
	
	/**
	 * Gets the sensitivity of neuron n
	 * 
	 * @param n the neuron to get the sensitivity of
	 * @return the value of the sensitivity
	 */
	public double getSensitivity(int n) {
		return layerSensitivities[n];
	}
	
	/**
	 * Gets the gradient array of this layer for data point d
	 * 
	 * @param d the data point index of the gradient
	 * @param n the neuron index of the gradient
	 * @param w the weight index of the gradient
	 * @return a 2D array containing the gradients of this layer
	 */
	public double getGradient(int d, int n, int w) {
		return layerGradients[d][n][w];		
	}
	
	/**
	 * Gets the entire array of sensitivities for this layer
	 * 
	 * @return an array of doubles that represents every sensitivity
	 * of this layer
	 */
	public double[] getSensitivities() {
		return layerSensitivities;
	}
	
	/**
	 * Gets the gradient array of this layer for data point d
	 * 
	 * @param d the data point to return the gradients for
	 * @return a 2D array containing the gradients of this layer
	 */
	public double[][] getGradients(int d) {
		return layerGradients[d];		
	}
	
	/**
	 * Sets the layer's weights to the incoming array
	 * 
	 * @param newWeights the array of weights to be set to
	 */
	public void setWeights(double[][] newWeights) {
		if((newWeights.length == layerWeights.length) &&
		   (newWeights[0].length == layerWeights[0].length)) {
			layerWeights = newWeights;
		}
		else {
			System.out.println("ERROR: Incoming weight vector does not " +
							   "match current weight size.");
			System.out.println("Incoming weight vector is [" + newWeights.length +
							   "][" + newWeights[0].length + "].");
			System.out.println("Old weight vector is [" + layerWeights.length +
							   "][" + layerWeights[0].length + "].");
		}
	}
	
	/**
	 * Sets the sensitivity of neuron n to s
	 * 
	 * @param n the neuron of the sensitivity to change
	 * @param s the value to change it to
	 * @return none
	 */
	public void setSensitivity(int n, double s) {
		layerSensitivities[n] = s;
	}
	
	/**
	 * Sets the gradient array of this layer for data point d
	 * 
	 * @param d the data point to return the gradients for
	 * @param n the neuron that this is a gradient for
	 * @param w the weight that this is a gradient for
	 * @param grad the new gradient
	 * @return none
	 */
	public void setGradient(int d, int n, int w, double grad) {
		layerGradients[d][n][w] = grad;
	}
	
	/**
	 * Returns a random double between -0.1 and 0.1
	 * This is used for initializing random weights.
	 * 
	 * @return a double between -0.1 and 0.1
	 */
	public double randomdouble() {
		double a = randomGen.nextDouble();
		double b = randomGen.nextInt();
		
		//multiplying two differently-gotten numbers
		//helps increase randomness
		
		a = a*b;
		
		/**
		 * Force positive values for out-of-bounds checks
		 */
		if(a < 0)
			a = -a;
		
		while(a > 0.6) {
			a /= 10;
		}
		
		/**
		 * Now that a is within accepted range, it can be
		 * negative if luck would have it
		 */
		if(randomGen.nextBoolean())
			a = -a;	
		
		return a;
	}
	
	private int layerNeuronCount;
	private int layerTrainingFunction;
	private int layerFunction;
	private double[] layerInput;
	private double[] layerOutput;
	private double[] layerSensitivities;
	/**
	 * layerGradients[d][x][y]
	 * d = data point number
	 * x = neuron number
	 * y = input line number
	 */
	private double[][][] layerGradients;
	/**
	 * layerWeights[x][y]
	 * x = neuron number
	 * y = input line number
	 */
	private double[][] layerWeights;
	
	private Random randomGen;
	
};
	
	public static class DataPoint {
		/**
		 * Class constructor.
		 * Creates a pairing of inputs and desired outputs to be used to train
		 * the network.
		 * 
		 * @param input an input point
		 * @param out the desired output for the given input
		 * @param inFile the name of the file which held this data point's info
		 */
		public DataPoint(double[] in, double[] out, String inFile) {
			dataInputVector = in;
			dataOutputVector = out;
			inputFile = inFile;
		}
		private double dataInputVector[];
		private double dataOutputVector[];
		private String inputFile;
		
		/**
		 * Returns the input vector of this data point
		 * 
		 * @return a double[] that represents the input
		 * vector for this point
		 */
		public double[] getInput() {
			return dataInputVector;
		}
		
		/**
		 * Returns the output vector of this data point
		 * 
		 * @return a double[] that represents the output
		 * vector for this point
		 */
		public double[] getOutput() {
			return dataOutputVector;
		}
		
		/**
		 * Returns the strings which represents the file
		 * that makes up this data point
		 * 
		 * @return a String[] of filenames
		 */
		public String getFile() {
			return inputFile;
		}
		
	};
	
	public static class ActivationFunction {
	
	/**
	 * Acts as the activation function for this layer. Takes the
	 * function input and converts it based on this layer's activation
	 * function.
	 * 
	 * @param n the input to this layer's activation function
	 * @return the output from this layer's activation function
	 */
	public static double function(int layerFunction, double n) {
		switch(layerFunction) {
		case HARDLIM:
			if(n >= 0)
				return 1;
			else
				return 0;
		case HARDLIMS:
			if(n >= 0)
				return 1;
			else
				return -1;
		case PURELIN:
			//do not use any function, this layer uses a
			//purelin function, where actual = n
			return n;
		case SATLIN:
			if(n < 0)
				return 0;
			else if(n > 1)
				return 1;
			else	// 0 <= n <= 1
				return n;
		case SATLINS:
			if(n < -1)
				return -1;
			else if(n > 1)
				return 1;
			else	// -1 <= n <= 1
				return n;
		case LOGSIG:
			return (1/(1 + Math.exp(-n)));
		case TANSIG:
			return ((Math.exp(n) - Math.exp(-n))/
					(Math.exp(n) + Math.exp(-n)));
		case POSLIN:
			if(n < 0)
				return 0;
			else
				return n;
		case COMPETE:
			//do not use any function, this layer uses a
			//competative function, which acts upon the entire
			//layer once the actual values have been computed
			//for each
			return n;
		default:
			return n;
		}
	}
	
	/**
	 * Returns the value of f'(n) where n is given and f is
	 * this layer's activation function.  Only gets called when network
	 * is in training, and thus should not ever use HARDLIM or HARDLIMS.
	 * 
	 * @param n the input to the function
	 * @return the derivative of the activation function applied
	 * to n
	 */
	public static double derivative(int layerFunction, double a) {
		switch(layerFunction) {
		case HARDLIM:
			System.out.println("HARDLIM function is not continuous, " +
							   "and has no derivative.\n\tUsing LOGSIG " +
							   "to train network.");
			return derivative(LOGSIG,a);
		case HARDLIMS:
			System.out.println("HARDLIMS function is not continuous, " +
							   "and has no derivative.\n\tUsing TANSIG " +
							   "to train network.");
			return derivative(TANSIG,a);
		case PURELIN:
			return 1;
		case SATLIN:
		case SATLINS:
		case LOGSIG:
			return (1-a)*a;
		case TANSIG:
			return 1-(a*a);
		case POSLIN:
		default:
			return a;
		}
	}
	
	/**
	 * The competative function, which selects the largest output
	 * from an entire layer's outputs, making it 1, and makes the rest 0
	 * 
	 * @param layerOutput the output vector for the layer
	 * @return the modified output, with only one non-zero output
	 */
	public static double[] competeFunction(double[] layerOutput) {
		int largestValueLocation = 0;
		double largestValue = layerOutput[0];
		
		int layerNeuronCount = layerOutput.length;
		for(int n = 1; n < layerNeuronCount; n++) {
			if(layerOutput[n] > largestValue) {
				largestValue = layerOutput[n];
				layerOutput[largestValueLocation] = 0;
				largestValueLocation = n;
			}
			else
				layerOutput[n] = 0;
		}
		layerOutput[largestValueLocation] = 1;
		
		return layerOutput;
	}
		
	/**
	 * Layer uses the Hard Limit function.
	 * <code> if n < 0, a = 0; if n >= 0, a = 1</code>
	 */
	public static final int HARDLIM = 0;
	/**
	 * Layer uses the Symmetrical Hard Limit function.
	 * <code> if n < 0, a = -1; if n >= 0, a = 1</code>
	 */
	public static final int HARDLIMS = 1;
	/**
	 * Layer uses the Linear function.
	 * <code> a = n </code>
	 */
	public static final int PURELIN = 2;
	/**
	 * Layer uses the Saturating Linear function.
	 * <code> if n < 0, a = 0; if 0 <= n <= 1, a = n; if n > 1, a = 1</code>
	 */
	public static final int SATLIN = 3;
	/**
	 * Layer uses the Symmetric Saturating Linear function.
	 * <code> if n < -1, a = -1; if -1 <= n <= 1, a = n; if n > 1, a = 1</code>
	 */
	public static final int SATLINS = 4;
	/**
	 * Layer uses the Log-Sigmoid function.
	 * <code> a = 1/(1+e^(-n)) </code>
	 */
	public static final int LOGSIG = 5;
	/**
	 * Layer uses the Hyperbolic Tangent Sigmoid function.
	 * <code> a = (e^n - e^(-n))/(e^n + e^(-n)) </code>
	 */
	public static final int TANSIG = 6;
	/**
	 * Layer uses the Positive Linear function.
	 * <code> if n < 0, a = 0; if n >= 0, a = n</code>
	 */
	public static final int POSLIN = 7;
	/**
	 * Layer uses the Competative function.
	 * <code> if n > all other n, a = 1;  else 0</code>
	 */
	public static final int COMPETE = 8;
	
};
	
	public static DataPoint[] parseInputFiles(String fileString) throws java.io.IOException {
		//FIXME
		final int FILE_LENGTH = 100*2;
		final int FEATURE_COUNT = 39;
		
		double[][] inputs;
		String[] filenames;
				
		int numFiles;
		int[] dataGoals;
		int numGroups;
		
		/**
		 * Get the list of files that contain the training data ready for reading
		 */
		FileReader fl = new FileReader(fileString);
		BufferedReader fileList = new BufferedReader(fl);
		
		String line = fileList.readLine();
		StringTokenizer tokenizer = new StringTokenizer(line);
					
		numFiles = Integer.parseInt(tokenizer.nextToken());
		numGroups = Integer.parseInt(tokenizer.nextToken());
		filenames = new String[numFiles];
		
		FileReader[] files = new FileReader[numFiles];
		BufferedReader[] dataFiles = new BufferedReader[numFiles];
		dataGoals = new int[numFiles];
		
		line = fileList.readLine();
		int fileCount = 0;
		while(line != null) {
			tokenizer = new StringTokenizer(line);
			
			dataGoals[fileCount] = Integer.parseInt(tokenizer.nextToken());
			
			filenames[fileCount] = tokenizer.nextToken();
			files[fileCount] = new FileReader(filenames[fileCount]);
			dataFiles[fileCount] = new BufferedReader(files[fileCount]);
			
			line = fileList.readLine();
			fileCount++;
		}
		
		fileList.close();
		
		/**
		 * Read in the data
		 */
		
		inputs = new double[numFiles][FILE_LENGTH*FEATURE_COUNT];

		for(int i = 0; i < numFiles; i++) {
			inputs[i] = parseDataFile(dataFiles[i],FILE_LENGTH*FEATURE_COUNT);
			
			dataFiles[i].close();
		}
		
		DataPoint[] points = new DataPoint[numFiles];
		
		
		for(int i = 0; i < numFiles; i++) {
			double[] groupVector = new double[numGroups];
			for(int j = 0; j < numGroups; j++)
				groupVector[j] = 0;
			
			switch(dataGoals[i]) {
			case 1:
				groupVector[0] = 1;
				break;
			case 2:
				groupVector[1] = 1;
				break;
			case 3:
				groupVector[2] = 1;
				break;
			case 4:
				groupVector[3] = 1;
				break;
			case 5:
				groupVector[4] = 1;
				break;
			case 6:
				groupVector[5] = 1;
				break;
			case 7:
				groupVector[6] = 1;
				break;
			case 8:
				groupVector[7] = 1;
				break;
			default:
				System.out.println(dataGoals[i] + " is not a supported category number.");
			}
			points[i] = new DataPoint(inputs[i], groupVector, filenames[i]);
		}
		
		return points;
	}
	
	public static double[] parseDataFile(BufferedReader inputFile, int inputSize)  throws java.io.IOException {
		double[] input = new double[inputSize];
		String line;
		StringTokenizer tokenizer;
		
		/**
		 * Skip the first 18 lines
		 */
		int lineNum = 0;
		for(; lineNum < 18; lineNum++) {
			inputFile.readLine();
		}
		line = inputFile.readLine();
		lineNum = 0;

		while((line != null) && lineNum < 200) {		//FIXME
			tokenizer = new StringTokenizer(line, ",");

			for(int j = 0; j < 39; j++) {				//FIXME
				input[lineNum*j] = Double.parseDouble(tokenizer.nextToken());
			}
			//skip the next line
			line = inputFile.readLine();
			line = inputFile.readLine();
			lineNum++;
		}
		
		return input;
	}
	
	/**
	 * The entry point function. Sends data points for the neural network to
	 * train itself on.
	 * 
	 * @param args command-line arguments [unused]
     * @return none
     */
    public static void main (String args[]) throws java.io.IOException {
		
		
		String trainList;
		String testList;
		
		String helpMsg = "Usage: java SpeakerID -t <trainingDataListFile> " +
						 "-r <testDataListFile> -L <LearingRate>";
		
		int numArgs = args.length;
		double learningRate;
		int iterations;
		
		/**
		 * Check to see if -train and -recognize (and their arguments) are both specified
		 */
		if (numArgs >= 4) {
			if(args[0].equalsIgnoreCase("-t")) {
				trainList = args[1];
				System.out.println("Training list file is: " + trainList);
				
				if(args[2].equalsIgnoreCase("-r")) {
					testList = args[3];
					System.out.println("Testing file is: " + testList);
					
					if(args[4].equalsIgnoreCase("-L")) {
						learningRate = Double.parseDouble(args[5]);
						System.out.println("Learning rate is: " + learningRate);
						
						if(args[6].equalsIgnoreCase("-I")) {
							iterations = Integer.parseInt(args[7]);
							System.out.println("Number of iterations to train: " + iterations);
						}
						else {
						System.out.println("Error: '" + args[6] + "' is not a valid flag.");
						System.out.println(helpMsg);
						return;
						}
					}
					else {
					System.out.println("Error: '" + args[4] + "' is not a valid flag.");
					System.out.println(helpMsg);
					return;
					}
				}
				else {
					System.out.println("Error: '" + args[2] + "' is not a valid flag.");
					System.out.println(helpMsg);
					return;
				}
			}
			else {
				System.out.println("Error: '" + args[0] + "' is not a valid flag.");
				System.out.println(helpMsg);
				return;
			}
		}	
		else {
			System.out.println(helpMsg);
			return;
		}
		
		System.out.println("\tLoading data files...");
		DataPoint[] trainingPoints = parseInputFiles(trainList);
		DataPoint[] testingPoints = parseInputFiles(testList);
		
		int numInputs = trainingPoints[0].getInput().length;
		int numGroups = trainingPoints[0].getOutput().length;
				
		/**
		 * layerSpec: for each layer in the network, layerSpec contains an entry.
		 * Each entry specification consists of an activation function and a neuron
		 * count
		 */
		int[][] layerSpec = {{ActivationFunction.COMPETE, numGroups*2},
							 {ActivationFunction.PURELIN, numGroups}};
		
		SpeakerID patternRec1 = new SpeakerID(numInputs,layerSpec,learningRate);
		
		/**
		 * Standard training
		 */
		System.out.println("\tTraining...");
		
		patternRec1.trainNetwork(trainingPoints,false,iterations);
		
		patternRec1.testNetwork(testingPoints);
		
		System.out.println("\n");
	}
}