MLP.cpp

/*
GRT MIT License
Copyright (c) <2012> <Nicholas Gillian, Media Lab, MIT>

Permission is hereby granted, free of charge, to any person obtaining a copy of this software
and associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial
portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

#define GRT_DLL_EXPORTS
#include "MLP.h"
#include "../../../CoreModules/Regressifier.h"

GRT_BEGIN_NAMESPACE

//Define the string that will be used to identify the object
const std::string MLP::id = "MLP";
std::string MLP::getId() { return MLP::id; }

//Register the MLP module with the base class
RegisterRegressifierModule< MLP > MLP::registerModule( MLP::getId() );

MLP::MLP() : Regressifier( MLP::getId() )
{
    inputLayerActivationFunction = Neuron::LINEAR;
    hiddenLayerActivationFunction = Neuron::TANH;
    outputLayerActivationFunction = Neuron::LINEAR;
    minNumEpochs = 1;
    numRestarts = 1;
    validationSetSize = 20; //20% of the training data will be set aside for the validation set
    trainingMode = ONLINE_GRADIENT_DESCENT;
    momentum = 0.5;
    gamma = 2.0;
    trainingError = 0;
    nullRejectionCoeff = 0.9;
    nullRejectionThreshold = 0;
    useValidationSet = true;
    randomiseTrainingOrder = false;
    useScaling = true;
    trained = false;
    initialized = false;
    classificationModeActive = false;
    useNullRejection = true;
    clear();
}

MLP::MLP(const MLP &rhs) : Regressifier( MLP::getId() )
{  
    *this = rhs;
}

MLP::~MLP(){
    clear();
}

MLP& MLP::operator=(const MLP &rhs){
    if( this != &rhs ){
        //MLP variables
        this->numInputNeurons = rhs.numInputNeurons;
        this->numHiddenNeurons = rhs.numHiddenNeurons;
        this->numOutputNeurons = rhs.numOutputNeurons;
        this->inputLayerActivationFunction = rhs.inputLayerActivationFunction;
        this->hiddenLayerActivationFunction = rhs.hiddenLayerActivationFunction;
        this->outputLayerActivationFunction = rhs.outputLayerActivationFunction;
        this->trainingMode = rhs.trainingMode;
        this->momentum = rhs.momentum;
        this->trainingError = rhs.trainingError;
        this->gamma = rhs.gamma;
        this->initialized = rhs.initialized;
        this->inputLayer = rhs.inputLayer;
        this->hiddenLayer = rhs.hiddenLayer;
        this->outputLayer = rhs.outputLayer;
        this->inputVectorRanges = rhs.inputVectorRanges;
        this->targetVectorRanges = rhs.targetVectorRanges;
        this->trainingErrorLog = rhs.trainingErrorLog;
        this->outputTargets = rhs.outputTargets;
        
        this->classificationModeActive = rhs.classificationModeActive;
        this->useNullRejection = rhs.useNullRejection;
        this->predictedClassLabel = rhs.predictedClassLabel;
        this->nullRejectionCoeff = rhs.nullRejectionCoeff;
        this->nullRejectionThreshold = rhs.nullRejectionThreshold;
        this->maxLikelihood = rhs.maxLikelihood;
        this->classLikelihoods = rhs.classLikelihoods;
        
        //Copy the base variables
        copyBaseVariables( (Regressifier*)&rhs );
    }
    return *this;
}

bool MLP::deepCopyFrom(const Regressifier *regressifier){
    
    if( regressifier == NULL ){
        errorLog << __GRT_LOG__ << " regressifier is NULL!" << std::endl;
        return false;
    }
    
    if( this->getId() != regressifier->getId() ){
        errorLog << __GRT_LOG__ << " regressifier is not the correct type!" << std::endl;
        return false;
    }
    
    *this = *dynamic_cast<const MLP*>(regressifier);
    
    return true;
}

//Classifier interface
bool MLP::train_(ClassificationData &trainingData){
    
    if( !initialized ){
        errorLog << __GRT_LOG__ << " The MLP has not been initialized!" << std::endl;
        return false;
    }
    
    if( trainingData.getNumDimensions() != numInputNeurons ){
        errorLog << __GRT_LOG__ << " The number of input dimensions in the training data (" << trainingData.getNumDimensions() << ") does not match that of the MLP (" << numInputNeurons << ")" << std::endl;
        return false;
    }
    if( trainingData.getNumClasses() != numOutputNeurons ){
        errorLog << __GRT_LOG__ << " The number of classes in the training data (" << trainingData.getNumClasses() << ") does not match that of the MLP (" << numOutputNeurons << ")" << std::endl;
        return false;
    }
    
    //Reformat the LabelledClassificationData as LabelledRegressionData
    RegressionData regressionData = trainingData.reformatAsRegressionData();
    
    //Flag that the MLP is being used for classification, not regression
    classificationModeActive = true;
    
    return trainModel(regressionData);
}

bool MLP::train_(RegressionData &trainingData){
    
    //Flag that the MLP is being used for regression, not classification
    classificationModeActive = false;
    
    return trainModel(trainingData);
}

//Classifier interface
bool MLP::predict_(VectorFloat &inputVector){
    
    if( !trained ){
        errorLog << __GRT_LOG__ << " Model not trained!" << std::endl;
        return false;
    }
    
    if( inputVector.getSize() != numInputNeurons ){
        errorLog << __GRT_LOG__ << " The size of the input Vector (" << inputVector.getSize() << ") does not match that of the number of input dimensions (" << numInputNeurons << ") " << std::endl;
        return false;
    }
    
    //Set the mapped data as the classLikelihoods
    regressionData = feedforward( inputVector );
    
    if( classificationModeActive ){
        
        //Estimate the class likelihoods
        const UINT K = (UINT)regressionData.getSize();
        classLikelihoods = regressionData;
        
        //Make sure all the values are greater than zero, we do this by finding the min value and adding this onto all the values
        Float minValue = Util::getMin( classLikelihoods );
        for(UINT i=0; i<K; i++){
            classLikelihoods[i] += minValue;
        }
        
        //Normalize the likelihoods so they sum to 1
        Float sum = Util::sum(classLikelihoods);
        if( sum > 0 ){
            for(UINT i=0; i<K; i++){
                classLikelihoods[i] /= sum;
            }
        }
        
        //Find the best value
        Float bestValue = classLikelihoods[0];
        UINT bestIndex = 0;
        for(UINT i=1; i<K; i++){
            if( classLikelihoods[i] > bestValue ){
                bestValue = classLikelihoods[i];
                bestIndex = i;
            }
        }
        
        //Set the maximum likelihood and predicted class label
        maxLikelihood = bestValue;
        predictedClassLabel = bestIndex+1;
        
        if( useNullRejection ){
            if( maxLikelihood < nullRejectionCoeff ){
                predictedClassLabel = 0;
            }
        }
    }
    
    return true;
}

bool MLP::init(const UINT numInputNeurons, const UINT numHiddenNeurons, const UINT numOutputNeurons){
    return init(numInputNeurons, numHiddenNeurons, numOutputNeurons, inputLayerActivationFunction, hiddenLayerActivationFunction, outputLayerActivationFunction );
}

bool MLP::init(const UINT numInputNeurons,
                const UINT numHiddenNeurons,
                const UINT numOutputNeurons,
                const Neuron::Type inputLayerActivationFunction,
                const Neuron::Type hiddenLayerActivationFunction,
                const Neuron::Type outputLayerActivationFunction){
    
    //Clear any previous models
    clear();
    
    //Initialize the random seed
    random.setSeed( (UINT)time(NULL) );
    
    if( numInputNeurons == 0 || numHiddenNeurons == 0 || numOutputNeurons == 0 ){
        if( numInputNeurons == 0 ){  errorLog << __GRT_LOG__ << " The number of input neurons is zero!" << std::endl; }
        if( numHiddenNeurons == 0 ){  errorLog << __GRT_LOG__ << " The number of hidden neurons is zero!" << std::endl; }
        if( numOutputNeurons == 0 ){  errorLog << __GRT_LOG__ << " The number of output neurons is zero!" << std::endl; }
        return false;
    }
    
    //Validate the activation functions
    if( !validateActivationFunction(inputLayerActivationFunction) || !validateActivationFunction(hiddenLayerActivationFunction) || !validateActivationFunction(outputLayerActivationFunction) ){
        errorLog << __GRT_LOG__ << " One Of The Activation Functions Failed The Validation Check" << std::endl;
        return false;
    }
    
    //Set the size of the MLP
    this->numInputNeurons = numInputNeurons;
    this->numHiddenNeurons = numHiddenNeurons;
    this->numOutputNeurons = numOutputNeurons;
    
    //Set the regression IO
    this->numInputDimensions = numInputNeurons;
    this->numOutputDimensions = numOutputNeurons;
    
    //Set the validation layers
    this->inputLayerActivationFunction = inputLayerActivationFunction;
    this->hiddenLayerActivationFunction = hiddenLayerActivationFunction;
    this->outputLayerActivationFunction = outputLayerActivationFunction;
    
    //Setup the neurons for each of the layers
    inputLayer.resize(numInputNeurons);
    hiddenLayer.resize(numHiddenNeurons);
    outputLayer.resize(numOutputNeurons);
    
    //Init the neuron memory for each of the layers
    for(UINT i=0; i<numInputNeurons; i++){
        inputLayer[i].init(1,inputLayerActivationFunction,random); //The input size for each input neuron will always be 1
        inputLayer[i].weights[0] = 1.0; //The weights for the input layer should always be 1
        inputLayer[i].bias = 0.0; //The bias for the input layer should always be 0
        inputLayer[i].gamma = gamma;
    }

    const Float hiddenLayerScaleFactor = 1.0/sqrt(numInputNeurons);
    const Float outputLayerScaleFactor = 1.0/sqrt(numHiddenNeurons);
    
    for(UINT i=0; i<numHiddenNeurons; i++){
        //The number of inputs to a neuron in the output layer will always match the number of input neurons
        hiddenLayer[i].init(numInputNeurons,hiddenLayerActivationFunction,random,-hiddenLayerScaleFactor,hiddenLayerScaleFactor);
        hiddenLayer[i].gamma = gamma;
    }
    
    for(UINT i=0; i<numOutputNeurons; i++){
        //The number of inputs to a neuron in the output layer will always match the number of hidden neurons
        outputLayer[i].init(numHiddenNeurons,outputLayerActivationFunction,random,-outputLayerScaleFactor,outputLayerScaleFactor);
        outputLayer[i].gamma = gamma;
    }
    
    initialized = true;
    
    return true;
}

bool MLP::clear(){
    
    //Clear the base class
    Regressifier::clear();
    
    numInputNeurons = 0;
    numHiddenNeurons = 0;
    numOutputNeurons = 0;
    inputLayer.clear();
    hiddenLayer.clear();
    outputLayer.clear();
    initialized = false;
    
    return true;
}

bool MLP::print() const{
    printNetwork();
    return true;
}

bool MLP::trainModel(RegressionData &trainingData){
    
    trained = false;
    
    if( !initialized ){
        errorLog << __GRT_LOG__ << " The MLP has not be initialized!" << std::endl;
        return false;
    }
    
    if( trainingData.getNumSamples() == 0 ){
        errorLog << __GRT_LOG__ << " The training data is empty!" << std::endl;
        return false;
    }
    
    //Create a validation dataset, if needed
    RegressionData validationData;
    if( useValidationSet ){
        validationData = trainingData.split( 100 - validationSetSize );
    }
    
    const UINT N = trainingData.getNumInputDimensions();
    const UINT T = trainingData.getNumTargetDimensions();
    
    if( N != numInputNeurons ){
        errorLog << __GRT_LOG__ << " The number of input dimensions in the training data (" << N << ") does not match that of the MLP (" << numInputNeurons << ")" << std::endl;
        return false;
    }
    if( T != numOutputNeurons ){
        errorLog << __GRT_LOG__ << " The number of target dimensions in the training data (" << T << ") does not match that of the MLP (" << numOutputNeurons << ")" << std::endl;
        return false;
    }
    
    //Set the Regressifier input and output dimensions
    numInputDimensions = numInputNeurons;
    numOutputDimensions = numOutputNeurons;

    //Set the target values that the output layer neurons should be scaled to
    setOutputTargets();
    
    //Scale the training and validation data, if needed
    if( useScaling ){
        //Find the ranges for the input data
        inputVectorRanges = trainingData.getInputRanges();
        
        //Find the ranges for the target data
        targetVectorRanges = trainingData.getTargetRanges();
        
        //Now scale the training data and the validation data if required
        trainingData.scale(inputVectorRanges,targetVectorRanges,outputTargets.minValue,outputTargets.maxValue);
        
        if( useValidationSet ){
            validationData.scale(inputVectorRanges,targetVectorRanges,outputTargets.minValue,outputTargets.maxValue);
        }
    }
    //If scaling is enabled then the training and validation data will be scaled - so turn it off so the we do not need to scale the data again
    //The actual scaling state will be reset at the end of traiing
    bool tempScalingState = useScaling;
    useScaling = false;
    
    //Setup the memory
    trainingErrorLog.clear();
    inputNeuronsOutput.resize(numInputNeurons);
    hiddenNeuronsOutput.resize(numHiddenNeurons);
    outputNeuronsOutput.resize(numOutputNeurons);
    deltaO.resize(numOutputNeurons);
    deltaH.resize(numHiddenNeurons);
    
    //Call the main training function
    switch( trainingMode ){
        case ONLINE_GRADIENT_DESCENT:
            if( classificationModeActive ){
                trained = trainOnlineGradientDescentClassification( trainingData, validationData );
            }else{
                trained = trainOnlineGradientDescentRegression( trainingData, validationData );
            }
        break;
        default:
        useScaling = tempScalingState;
        errorLog << __GRT_LOG__ << " Uknown training mode!" << std::endl;
        return false;
        break;
    }
    
    //Reset the scaling state so the prediction data will be scaled if needed
    useScaling = tempScalingState;
    
    return trained;
}

bool MLP::trainOnlineGradientDescentClassification(const RegressionData &trainingData,const RegressionData &validationData){
    
    const UINT M = trainingData.getNumSamples();
    const UINT T = trainingData.getNumTargetDimensions();
    const UINT numTestingExamples = useValidationSet ? validationData.getNumSamples() : M;
    
    //Setup the training loop
    MLP bestNetwork;
    totalSquaredTrainingError = 0;
    rmsTrainingError = 0;
    rmsValidationError = 0;
    trainingError = 0;
    bool keepTraining = true;
    UINT epoch = 0;
    UINT bestIter = 0;
    UINT bestIndex = 0;
    UINT classLabel = 0;
    Float lRate = learningRate;
    Float lMomentum = momentum;
    Float error = 0;
    Float lastError = 0;
    Float accuracy = 0;
    Float trainingSetAccuracy = 0;
    Float trainingSetTotalSquaredError = 0;
    Float bestError = grt_numeric_limits< Float >::max();
    Float bestTSError = grt_numeric_limits< Float >::max();
    Float bestRMSError = grt_numeric_limits< Float >::max();
    Float bestAccuracy = 0;
    Float delta = 0;
    Float backPropError = 0;
    Float bestValue = 0;
    VectorFloat y;
    Vector< UINT > indexList(M);
    Vector< VectorFloat > tempTrainingErrorLog;
    TrainingResult result;
    trainingResults.reserve(M);
    
    //Reset the indexList, this is used to randomize the order of the training examples, if needed
    for(UINT i=0; i<M; i++) indexList[i] = i;
    
    for(UINT iter=0; iter<numRestarts; iter++){
        
        epoch = 0;
        keepTraining = true;
        tempTrainingErrorLog.clear();
        
        //Randomise the start values of the neurons
        init(numInputNeurons,numHiddenNeurons,numOutputNeurons,inputLayerActivationFunction,hiddenLayerActivationFunction,outputLayerActivationFunction);
        
        if( randomiseTrainingOrder ){
            for(UINT i=0; i<M; i++){
                SWAP(indexList[ i ], indexList[ random.getRandomNumberInt(0, M) ]);
            }
        }
        
        while( keepTraining ){
            
            //Perform one training epoch
            accuracy = 0;
            totalSquaredTrainingError = 0;
            
            for(UINT i=0; i<M; i++){
                //Get the i'th training and target Vectors
                const VectorFloat &trainingExample = trainingData[ indexList[i] ].getInputVector();
                const VectorFloat &targetVector = trainingData[ indexList[i] ].getTargetVector();
                
                //Perform the back propagation
                backPropError = back_prop(trainingExample,targetVector,lRate,lMomentum);
                
                //debugLog << "i: " << i << " backPropError: " << backPropError << std::endl;
                
                if( isNAN(backPropError) ){
                    keepTraining = false;
                    errorLog << __GRT_LOG__ << " NaN found in back propagation error, training index: " << indexList[i] << std::endl;
                    return false;
                }
                
                //Compute the error for the i'th example
                if( classificationModeActive ){
                    y = feedforward( trainingExample );
                    
                    //Get the class label
                    bestValue = targetVector[0];
                    bestIndex = 0;
                    for(UINT i=1; i<targetVector.size(); i++){
                        if( targetVector[i] > bestValue ){
                            bestValue = targetVector[i];
                            bestIndex = i;
                        }
                    }
                    classLabel = bestIndex + 1;
                    
                    //Get the predicted class label
                    bestValue = y[0];
                    bestIndex = 0;
                    for(UINT i=1; i<numOutputNeurons; i++){
                        if( y[i] > bestValue ){
                            bestValue = y[i];
                            bestIndex = i;
                        }
                    }
                    predictedClassLabel = bestIndex+1;
                    
                    if( classLabel == predictedClassLabel ){
                        accuracy++;
                    }
                    
                }else{
                    totalSquaredTrainingError += backPropError; //The backPropError is already squared
                }
            }
            
            if( checkForNAN() ){
                keepTraining = false;
                errorLog << __GRT_LOG__ << " NaN found in weights at " << epoch << std::endl;
                break;
            }
            
            //Compute the error over all the training/validation examples
            if( useValidationSet ){
                trainingSetAccuracy = accuracy;
                trainingSetTotalSquaredError = totalSquaredTrainingError;
                accuracy = 0;
                totalSquaredTrainingError = 0;
                
                //Iterate over the validation samples
                UINT numValidationSamples = validationData.getNumSamples();
                for(UINT i=0; i<numValidationSamples; i++){
                    const VectorFloat &inputVector = validationData[i].getInputVector();
                    const VectorFloat &targetVector = validationData[i].getTargetVector();
                    
                    y = feedforward( inputVector );
                    
                    if( classificationModeActive ){
                        //Get the class label
                        bestValue = targetVector[0];
                        bestIndex = 0;
                        for(UINT i=1; i<numInputNeurons; i++){
                            if( targetVector[i] > bestValue ){
                                bestValue = targetVector[i];
                                bestIndex = i;
                            }
                        }
                        classLabel = bestIndex + 1;
                        
                        //Get the predicted class label
                        bestValue = y[0];
                        bestIndex = 0;
                        for(UINT i=1; i<numOutputNeurons; i++){
                            if( y[i] > bestValue ){
                                bestValue = y[i];
                                bestIndex = i;
                            }
                        }
                        predictedClassLabel = bestIndex+1;
                        
                        if( classLabel == predictedClassLabel ){
                            accuracy++;
                        }
                        
                    }else{
                        //Update the total squared error
                        for(UINT j=0; j<T; j++){
                            totalSquaredTrainingError += SQR( targetVector[j]-y[j] );
                        }
                    }
                }
                
                accuracy = (accuracy/Float(numValidationSamples))*Float(numValidationSamples);
                rmsValidationError = sqrt( totalSquaredTrainingError / Float(numValidationSamples) );
                
            }else{//We are not using a validation set
                accuracy = (accuracy/Float(M))*Float(M);
                rmsTrainingError = sqrt( totalSquaredTrainingError / Float(M) );
            }
            
            //Store the errors
            VectorFloat temp(2);
            temp[0] = 100.0 - trainingSetAccuracy;
            temp[1] = 100.0 - accuracy;
            tempTrainingErrorLog.push_back( temp );
            
            error = 100.0 - accuracy;
            
            //Store the training results
            result.setClassificationResult(iter,accuracy,this);
            trainingResults.push_back( result );
            
            delta = fabs( error - lastError );
            
            trainingLog << "Random Training Iteration: " << iter+1 << " Epoch: " << epoch << " Error: " << error << " Delta: " << delta << std::endl;
            
            //Check to see if we should stop training
            if( ++epoch >= maxNumEpochs ){
                keepTraining = false;
            }
            if( delta <= minChange && epoch >= minNumEpochs ){
                keepTraining = false;
            }
            
            //Update the last error
            lastError = error;
            
            //Notify any observers of the new training result
            trainingResultsObserverManager.notifyObservers( result );
            
        }//End of While( keepTraining )
        
        if( lastError < bestError ){
            bestIter = iter;
            bestError = lastError;
            bestTSError = totalSquaredTrainingError;
            bestRMSError = rmsTrainingError;
            bestAccuracy = accuracy;
            bestNetwork = *this;
            trainingErrorLog = tempTrainingErrorLog;
        }
        
    }//End of For( numRestarts )
    
    trainingLog << "Best Accuracy: " << bestAccuracy << " in Random Training Iteration: " << bestIter+1 << std::endl;
    
    //Check to make sure the best network has not got any NaNs in it
    if( checkForNAN() ){
        errorLog << __GRT_LOG__ << " NAN Found!" << std::endl;
        return false;
    }
    
    //Set the MLP model to the model that best during training
    *this = bestNetwork;
    trainingError = bestAccuracy;
    
    //Compute the rejection threshold
    if( useNullRejection ){
        
        Float averageValue = 0;
        VectorFloat classificationPredictions, inputVector, targetVector;
        
        for(UINT i=0; i<numTestingExamples; i++){
            inputVector = useValidationSet ? validationData[i].getInputVector() : trainingData[i].getInputVector();
            targetVector = useValidationSet ? validationData[i].getTargetVector() : trainingData[i].getTargetVector();
            
            //Make the prediction
            y = feedforward( inputVector );
            
            //Get the class label
            bestValue = targetVector[0];
            bestIndex = 0;
            for(UINT i=1; i<targetVector.size(); i++){
                if( targetVector[i] > bestValue ){
                    bestValue = targetVector[i];
                    bestIndex = i;
                }
            }
            classLabel = bestIndex + 1;
            
            //Get the predicted class label
            bestValue = y[0];
            bestIndex = 0;
            for(UINT i=1; i<y.size(); i++){
                if( y[i] > bestValue ){
                    bestValue = y[i];
                    bestIndex = i;
                }
            }
            predictedClassLabel = bestIndex+1;
            
            //Only add the max value if the prediction is correct
            if( classLabel == predictedClassLabel ){
                classificationPredictions.push_back( bestValue );
                averageValue += bestValue;
            }
        }
        
        averageValue /= Float(classificationPredictions.size());
        Float stdDev = 0;
        for(UINT i=0; i<classificationPredictions.size(); i++){
            stdDev += SQR(classificationPredictions[i]-averageValue);
        }
        stdDev = sqrt( stdDev / Float(classificationPredictions.size()-1) );
        
        nullRejectionThreshold = averageValue-(stdDev*nullRejectionCoeff);
    }
    
    //Return true to flag that the model was trained OK
    return true;
}

bool MLP::trainOnlineGradientDescentRegression(const RegressionData &trainingData,const RegressionData &validationData){
    
    const UINT M = trainingData.getNumSamples();
    const UINT T = trainingData.getNumTargetDimensions();
    const UINT numValidationSamples = useValidationSet ? validationData.getNumSamples() : M;
    
    //Setup the training loop
    bool keepTraining = true;
    UINT epoch = 0;
    Float alpha = learningRate;
    Float beta = momentum;
    UINT bestIter = 0;
    MLP bestNetwork;
    totalSquaredTrainingError = 0;
    rmsTrainingError = 0;
    trainingError = 0;
    Float error = 0;
    Float lastError = 0;
    Float bestError = grt_numeric_limits< Float >::max();
    Float bestTSError = grt_numeric_limits< Float >::max();
    Float bestRMSError = grt_numeric_limits< Float >::max();
    Float delta = 0;
    Vector< UINT > indexList(M);
    Vector< VectorFloat > tempTrainingErrorLog;
    TrainingResult result;
    trainingResults.reserve(M);
    
    //Reset the indexList, this is used to randomize the order of the training examples, if needed
    for(UINT i=0; i<M; i++) indexList[i] = i;
    
    for(UINT iter=0; iter<numRestarts; iter++){
        
        epoch = 0;
        keepTraining = true;
        tempTrainingErrorLog.clear();
        
        //Randomise the start values of the neurons
        init(numInputNeurons,numHiddenNeurons,numOutputNeurons,inputLayerActivationFunction,hiddenLayerActivationFunction,outputLayerActivationFunction);
        
        if( randomiseTrainingOrder ){
            std::random_shuffle(indexList.begin(), indexList.end());
        }
        
        while( keepTraining ){
            
            //Perform one training epoch
            totalSquaredTrainingError = 0;
            rmsTrainingError = 0;
            rmsValidationError = 0;
            
            for(UINT i=0; i<M; i++){

                //Get the i'th training and target Vectors
                const UINT randomIndex = indexList[i];
                const VectorFloat &trainingExample = trainingData[ randomIndex ].getInputVector();
                const VectorFloat &targetVector = trainingData[ randomIndex ].getTargetVector();
                
                //Perform the back propagation
                Float backPropError = back_prop(trainingExample,targetVector,alpha,beta);
                 
                if( isNAN(backPropError) ){
                    keepTraining = false;
                    errorLog << __GRT_LOG__ << " NaN found in back propagation error, epoch: " << epoch << " training iter: " << i << " random index: " << indexList[ randomIndex ] << std::endl;
                    return false;
                }

                //Compute the error for the i'th example
                totalSquaredTrainingError += backPropError; //The backPropError is already squared
            }
            
            if( checkForNAN() ){
                keepTraining = false;
                errorLog << __GRT_LOG__ << " NaN found in weights at epoch " << epoch << std::endl;
                return false;
            }
                
            //Compute the rms error on the training set
            rmsTrainingError = sqrt( totalSquaredTrainingError / Float(M) );
            
            //Compute the error over all the training/validation examples
            if( useValidationSet ){
                
                //Iterate over the validation samples
                for(UINT n=0; n<numValidationSamples; n++){
                    const VectorFloat &trainingExample = validationData[n].getInputVector();
                    const VectorFloat &targetVector = validationData[n].getTargetVector();
                    
                    VectorFloat y = feedforward(trainingExample);
                    
                    //Update the error
                    Float error = 0;
                    for(UINT j=0; j<T; j++){
                        error += SQR( targetVector[j]-y[j] );
                    }
                    rmsValidationError += sqrt( error );
                }
                
                rmsValidationError = sqrt( rmsValidationError / Float(numValidationSamples) );
            }
            
            //Store the errors
            VectorFloat temp(2);
            temp[0] = rmsTrainingError;
            temp[1] = rmsValidationError;
            tempTrainingErrorLog.push_back( temp );
            
            error = rmsTrainingError;
            
            //Store the training results
            result.setRegressionResult(iter,rmsTrainingError,rmsValidationError,this);
            trainingResults.push_back( result );
            
            delta = fabs( error - lastError );
            
            trainingLog << "Random Training Iteration: " << iter+1 << " Epoch: " << epoch << " Rms Training Error: " << rmsTrainingError;
            if( useValidationSet ) trainingLog << " Rms Validation Error: " << rmsValidationError;
            trainingLog << " Delta: " << delta << std::endl;
            
            //Check to see if we should stop training
            if( ++epoch >= maxNumEpochs ){
                keepTraining = false;
            }
            if( delta <= minChange && epoch >= minNumEpochs ){
                keepTraining = false;
            }
            
            //Update the last error
            lastError = error;
            
            //Notify any observers of the new training result
            trainingResultsObserverManager.notifyObservers( result );
            
        }//End of While( keepTraining )
        
        //Check to see if this is the best model so far
        if( lastError < bestError ){
            bestIter = iter;
            bestError = lastError;
            bestTSError = totalSquaredTrainingError;
            bestRMSError = rmsTrainingError;
            bestNetwork = *this;
            trainingErrorLog = tempTrainingErrorLog;
        }
        
    }//End of For( numRestarts )
    
    trainingLog << "Best Rms Error: " << bestRMSError << " in Random Training Iteration: " << bestIter+1 << std::endl;
    
    //Check to make sure the best network has not got any NaNs in it
    if( checkForNAN() ){
        errorLog << __GRT_LOG__ << " NAN Found!" << std::endl;
        return false;
    }
    
    //Set the MLP model to the model that best during training
    *this = bestNetwork;
    trainingError = bestRMSError;
    
    //Return true to flag that the model was trained OK
    return true;
}

Float MLP::back_prop(const VectorFloat &trainingExample,const VectorFloat &targetVector,const Float learningRate,const Float learningMomentum){
    
    UINT i,j,k = 0;
    Float update = 0;
    Float error = 0;
    Float sqrError = 0;
    
    //Forward propagation based on the current weights
    feedforward(trainingExample,inputNeuronsOutput,hiddenNeuronsOutput,outputNeuronsOutput);
 
    //Compute the error of the output layer: the derivative of the output neuron, times the error of the output
    for(k=0; k<numOutputNeurons; k++){
        error = targetVector[k]-outputNeuronsOutput[k];
        sqrError += SQR( error );
        deltaO[k] = outputLayer[k].getDerivative( outputNeuronsOutput[k] ) * error;
    }
    sqrError = sqrt( sqrError );
    
    //Compute the error of the hidden layer: the derivative of the hidden neuron, times the total error of the hidden output 
    for(j=0; j<numHiddenNeurons; j++){
        error = 0;
        for(k=0; k<numOutputNeurons; k++){
            error += outputLayer[k].weights[j] * deltaO[k];
        }
        deltaH[j] = hiddenLayer[j].getDerivative( hiddenNeuronsOutput[j] ) * error;
    }

    //Update the output weights, new weight = old weight + (learningRate * change) + (momenutum * previousChange)
    for(j=0; j<numHiddenNeurons; j++){
        for(k=0; k<numOutputNeurons; k++){
            update = deltaO[k] * hiddenNeuronsOutput[j];
            outputLayer[k].weights[j] += learningRate*update + learningMomentum*outputLayer[k].previousUpdate[j];
            outputLayer[k].previousUpdate[j] = update;
        }
    }

    //Update the hidden weights, new weight = old weight + (learningRate * change) + (momenutum * previousChange)
    for(i=0; i<numInputNeurons; i++){
        for(j=0; j<numHiddenNeurons; j++){
            update = deltaH[j] * inputNeuronsOutput[i];
            hiddenLayer[j].weights[i] += learningRate*update + learningMomentum*hiddenLayer[j].previousUpdate[i];
            hiddenLayer[j].previousUpdate[i] = update;
        }
    }
    
    //Update the output bias, new weight = old weight + (learningRate * change) + (momenutum * previousChange)
    for(k=0; k<numOutputNeurons; k++){
        //Compute the update
        update = learningRate*deltaO[k] + learningMomentum*outputLayer[k].previousBiasUpdate;
        
        //Update the bias
        outputLayer[k].bias += update;
        
        //Store the update
        outputLayer[k].previousBiasUpdate = update;
    }

    //Update the hidden bias, new weight = old weight + (learningRate * change) + (momenutum * previousChange)
    for(j=0; j<numHiddenNeurons; j++){
        //Compute the update
        update = learningRate*deltaH[j] + learningMomentum*hiddenLayer[j].previousBiasUpdate;
        
        //Update the bias
        hiddenLayer[j].bias += update;
        
        //Store the update
        hiddenLayer[j].previousBiasUpdate = update;
    }
    
    //Return the squared error between the output of the network and the target Vector
    return sqrError;
}

VectorFloat MLP::feedforward(VectorFloat trainingExample){
    
    if( inputNeuronsOutput.size() != numInputNeurons ) inputNeuronsOutput.resize(numInputNeurons,0);
    if( hiddenNeuronsOutput.size() != numHiddenNeurons ) hiddenNeuronsOutput.resize(numHiddenNeurons,0);
    if( outputNeuronsOutput.size() != numOutputNeurons ) outputNeuronsOutput.resize(numOutputNeurons,0);

    UINT i,j,k=0;
    
    //Scale the input vector if required
    if( useScaling ){
        for(i=0; i<numInputNeurons; i++){
            trainingExample[i] = scale(trainingExample[i],inputVectorRanges[i].minValue,inputVectorRanges[i].maxValue,outputTargets.minValue,outputTargets.maxValue);
        }
    }
    
    //Input layer
    VectorFloat input(1);
    for(i=0; i<numInputNeurons; i++){
        input[0] = trainingExample[i];
        inputNeuronsOutput[i] = inputLayer[i].fire( input );
    }
    
    //Hidden Layer
    for(j=0; j<numHiddenNeurons; j++){
        hiddenNeuronsOutput[j] = hiddenLayer[j].fire( inputNeuronsOutput );
    }
    
    //Output Layer
    for(k=0; k<numOutputNeurons; k++){
        outputNeuronsOutput[k] = outputLayer[k].fire( hiddenNeuronsOutput );
    }
    
    //Scale the output vector if required
    if( useScaling ){
        for(k=0; k<numOutputNeurons; k++){
            outputNeuronsOutput[k] = scale(outputNeuronsOutput[k],outputTargets.minValue,outputTargets.maxValue,targetVectorRanges[k].minValue,targetVectorRanges[k].maxValue);
        }
    }
    
    return outputNeuronsOutput;
}

void MLP::feedforward(const VectorFloat &data,VectorFloat &inputNeuronsOutput,VectorFloat &hiddenNeuronsOutput,VectorFloat &outputNeuronsOutput){
    
    if( inputNeuronsOutput.size() != numInputNeurons ) inputNeuronsOutput.resize(numInputNeurons,0);
    if( hiddenNeuronsOutput.size() != numHiddenNeurons ) hiddenNeuronsOutput.resize(numHiddenNeurons,0);
    if( outputNeuronsOutput.size() != numOutputNeurons ) outputNeuronsOutput.resize(numOutputNeurons,0);

    UINT i,j,k=0;
    
    //Input layer
    VectorFloat input(1);
    for(i=0; i<numInputNeurons; i++){
        input[0] = data[i];
        inputNeuronsOutput[i] = inputLayer[i].fire( input );
    }
    
    //Hidden Layer
    for(j=0; j<numHiddenNeurons; j++){
        hiddenNeuronsOutput[j] = hiddenLayer[j].fire( inputNeuronsOutput );
    }
    
    //Output Layer
    for(k=0; k<numOutputNeurons; k++){
        outputNeuronsOutput[k] = outputLayer[k].fire( hiddenNeuronsOutput );
    }
    
}

void MLP::printNetwork() const{
    std::cout<<"***************** MLP *****************\n";
    std::cout<<"NumInputNeurons: "<<numInputNeurons<< std::endl;
    std::cout<<"NumHiddenNeurons: "<<numHiddenNeurons<< std::endl;
    std::cout<<"NumOutputNeurons: "<<numOutputNeurons<< std::endl;

    std::cout << "ScalingEnabled: " << useScaling << std::endl;

    if( useScaling ){
        std::cout << "InputRanges: " << std::endl;
        for(UINT i=0; i<numInputNeurons; i++){
            std::cout << "Input: " << i << "\t" << inputVectorRanges[i].minValue << "\t" << inputVectorRanges[i].maxValue << std::endl;
        }

        std::cout << "OutputRanges: " << std::endl;
        for(UINT i=0; i<numOutputNeurons; i++){
            std::cout << "Output: " << i << "\t" << targetVectorRanges[i].minValue << "\t" << targetVectorRanges[i].maxValue << std::endl;
        }
    }
    
    std::cout<<"InputWeights:\n";
    for(UINT i=0; i<numInputNeurons; i++){
        std::cout<<"Neuron: "<<i<<" Bias: " << inputLayer[i].bias << " Weights: ";
        for(UINT j=0; j<inputLayer[i].weights.getSize(); j++){
            std::cout << inputLayer[i].weights[j] << "\t";
        } std::cout << std::endl;
    }
    
    std::cout<<"HiddenWeights:\n";
    for(UINT i=0; i<numHiddenNeurons; i++){
        std::cout<<"Neuron: "<<i<<" Bias: " << hiddenLayer[i].bias << " Weights: ";
        for(UINT j=0; j<hiddenLayer[i].weights.getSize(); j++){
            std::cout << hiddenLayer[i].weights[j] << "\t";
        } std::cout << std::endl;
    }
    
    std::cout<<"OutputWeights:\n";
    for(UINT i=0; i<numOutputNeurons; i++){
        std::cout<<"Neuron: "<<i<<" Bias: " << outputLayer[i].bias << " Weights: ";
        for(UINT j=0; j<outputLayer[i].weights.getSize(); j++){
            std::cout << outputLayer[i].weights[j] << "\t";
        } std::cout << std::endl;
    }
    
}

bool MLP::checkForNAN() const{
    
    UINT N = 0;
    for(UINT i=0; i<numInputNeurons; i++){
        if( isNAN(inputLayer[i].bias) ) return true;
        N = inputLayer[i].weights.getSize();
        for(UINT j=0; j<N; j++){
            if( isNAN(inputLayer[i].weights[j]) ) return true;
        }
    }
    
    for(UINT i=0; i<numHiddenNeurons; i++){
        if( isNAN(hiddenLayer[i].bias) ) return true;
        N = hiddenLayer[i].weights.getSize();
        for(UINT j=0; j<N; j++){
            if( isNAN(hiddenLayer[i].weights[j]) ) return true;
        }
    }
    
    for(UINT i=0; i<numOutputNeurons; i++){
        if( isNAN(outputLayer[i].bias) ) return true;
        N = outputLayer[i].weights.getSize();
        for(UINT j=0; j<N; j++){
            if( isNAN(outputLayer[i].weights[j]) ) return true;
        }
    }
    
    return false;
}

bool inline MLP::isNAN(const Float &v) const{
    if( v != v ) return true;
    return false;
}

bool MLP::save( std::fstream &file ) const{
    
    if( !file.is_open() ){
        errorLog << __GRT_LOG__ << " File is not open!" << std::endl;
        return false;
    }
    
    file << "GRT_MLP_FILE_V2.0\n";
    
    //Write the regressifier settings to the file
    if( !Regressifier::saveBaseSettingsToFile(file) ){
        errorLog << __GRT_LOG__ << " Failed to save Regressifier base settings to file!" << std::endl;
        return false;
    }
    
    file << "NumInputNeurons: "<<numInputNeurons<< std::endl;
    file << "NumHiddenNeurons: "<<numHiddenNeurons<< std::endl;
    file << "NumOutputNeurons: "<<numOutputNeurons<< std::endl;
    file << "InputLayerActivationFunction: " <<activationFunctionToString(inputLayerActivationFunction)<< std::endl;
    file << "HiddenLayerActivationFunction: " <<activationFunctionToString(hiddenLayerActivationFunction)<< std::endl;
    file << "OutputLayerActivationFunction: " <<activationFunctionToString(outputLayerActivationFunction)<< std::endl;
    file << "NumRandomTrainingIterations: " << numRestarts << std::endl;
    file << "Momentum: " << momentum << std::endl;
    file << "Gamma: " << gamma << std::endl;
    file << "ClassificationMode: " << classificationModeActive << std::endl;
    file << "UseNullRejection: " << useNullRejection << std::endl;
    file << "RejectionThreshold: " << nullRejectionThreshold << std::endl;
    
    if( trained ){
        file << "InputLayer: \n";
        for(UINT i=0; i<numInputNeurons; i++){
            file << "InputNeuron: " << i+1 << std::endl;
            file << "NumInputs: " << inputLayer[i].numInputs << std::endl;
            file << "Bias: " << inputLayer[i].bias << std::endl;
            file << "Gamma: " << inputLayer[i].gamma << std::endl;
            file << "Weights: " << std::endl;
            for(UINT j=0; j<inputLayer[i].numInputs; j++){
                file << inputLayer[i].weights[j] << "\t";
            }
            file << std::endl;
        }
        file << "\n";
        
        file << "HiddenLayer: \n";
        for(UINT i=0; i<numHiddenNeurons; i++){
            file << "HiddenNeuron: " << i+1 << std::endl;
            file << "NumInputs: " << hiddenLayer[i].numInputs << std::endl;
            file << "Bias: " << hiddenLayer[i].bias << std::endl;
            file << "Gamma: " << hiddenLayer[i].gamma << std::endl;
            file << "Weights: " << std::endl;
            for(UINT j=0; j<hiddenLayer[i].numInputs; j++){
                file << hiddenLayer[i].weights[j] << "\t";
            }
            file << std::endl;
        }
        file << "\n";
        
        file << "OutputLayer: \n";
        for(UINT i=0; i<numOutputNeurons; i++){
            file << "OutputNeuron: " << i+1 << std::endl;
            file << "NumInputs: " << outputLayer[i].numInputs << std::endl;
            file << "Bias: " << outputLayer[i].bias << std::endl;
            file << "Gamma: " << outputLayer[i].gamma << std::endl;
            file << "Weights: " << std::endl;
            for(UINT j=0; j<outputLayer[i].numInputs; j++){
                file << outputLayer[i].weights[j] << "\t";
            }
            file << std::endl;
        }
    }
    
    return true;
}

bool MLP::load( std::fstream &file ){
    
    std::string activationFunction;
    
    //Clear any previous models
    clear();
    
    if( !file.is_open() ){
        errorLog << __GRT_LOG__ << " File is not open!" << std::endl;
        return false;
    }
    
    std::string word;
    
    file >> word;
    
    //See if we should load a legacy file
    if( word == "GRT_MLP_FILE_V1.0" ){
        return loadLegacyModelFromFile( file );
    }
    
    //Check to make sure this is a file with the MLP File Format
    if( word != "GRT_MLP_FILE_V2.0" ){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find file header!" << std::endl;
        return false;
    }
    
    //Load the base settings from the file
    if( !Regressifier::loadBaseSettingsFromFile( file ) ){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to load regressifier base settings from file!" << std::endl;
        return false;
    }
    
    file >> word;
    if(word != "NumInputNeurons:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find NumInputNeurons!" << std::endl;
        return false;
    }
    file >> numInputNeurons;
    numInputDimensions = numInputNeurons;
    
    file >> word;
    if(word != "NumHiddenNeurons:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find NumHiddenNeurons!" << std::endl;
        return false;
    }
    file >> numHiddenNeurons;
    
    file >> word;
    if(word != "NumOutputNeurons:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find NumOutputNeurons!" << std::endl;
        return false;
    }
    file >> numOutputNeurons;
    
    file >> word;
    if(word != "InputLayerActivationFunction:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find InputLayerActivationFunction!" << std::endl;
        return false;
    }
    file >> activationFunction;
    inputLayerActivationFunction = activationFunctionFromString(activationFunction);
    
    file >> word;
    if(word != "HiddenLayerActivationFunction:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find HiddenLayerActivationFunction!" << std::endl;
        return false;
    }
    file >> activationFunction;
    hiddenLayerActivationFunction = activationFunctionFromString(activationFunction);
    
    file >> word;
    if(word != "OutputLayerActivationFunction:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find OutputLayerActivationFunction!" << std::endl;
        return false;
    }
    file >> activationFunction;
    outputLayerActivationFunction = activationFunctionFromString(activationFunction);
    
    file >> word;
    if(word != "NumRandomTrainingIterations:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find NumRandomTrainingIterations!" << std::endl;
        return false;
    }
    file >> numRestarts;
    
    file >> word;
    if(word != "Momentum:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find Momentum!" << std::endl;
        return false;
    }
    file >> momentum;
    
    file >> word;
    if(word != "Gamma:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find Gamma!" << std::endl;
        return false;
    }
    file >> gamma;
    
    file >> word;
    if(word != "ClassificationMode:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find ClassificationMode!" << std::endl;
        return false;
    }
    file >> classificationModeActive;
    
    file >> word;
    if(word != "UseNullRejection:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find UseNullRejection!" << std::endl;
        return false;
    }
    file >> useNullRejection;
    
    file >> word;
    if(word != "RejectionThreshold:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find RejectionThreshold!" << std::endl;
        return false;
    }
    file >> nullRejectionThreshold;
    
    if( trained ) initialized = true;
    else init(numInputNeurons,numHiddenNeurons,numOutputNeurons);
        
    if( trained ){
        
        //Resize the layers
        inputLayer.resize( numInputNeurons );
        hiddenLayer.resize( numHiddenNeurons );
        outputLayer.resize( numOutputNeurons );
        
        //Load the neuron data
        file >> word;
        if(word != "InputLayer:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find InputLayer!" << std::endl;
            return false;
        }
        
        for(UINT i=0; i<numInputNeurons; i++){
            UINT tempNeuronID = 0;
            
            file >> word;
            if(word != "InputNeuron:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find InputNeuron!" << std::endl;
                return false;
            }
            file >> tempNeuronID;
            
            if( tempNeuronID != i+1 ){
                file.close();
                errorLog << __GRT_LOG__ << " InputNeuron ID does not match!" << std::endl;
                return false;
            }
            
            file >> word;
            if(word != "NumInputs:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find NumInputs!" << std::endl;
                return false;
            }
            file >> inputLayer[i].numInputs;
            
            //Resize the buffers
            inputLayer[i].weights.resize( inputLayer[i].numInputs );
            
            file >> word;
            if(word != "Bias:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find Bias!" << std::endl;
                return false;
            }
            file >> inputLayer[i].bias;
            
            file >> word;
            if(word != "Gamma:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find Gamma!" << std::endl;
                return false;
            }
            file >> inputLayer[i].gamma;
            
            file >> word;
            if(word != "Weights:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find Weights!" << std::endl;
                return false;
            }
            
            for(UINT j=0; j<inputLayer[i].numInputs; j++){
                file >> inputLayer[i].weights[j];
            }
        }
        
        //Load the Hidden Layer
        file >> word;
        if(word != "HiddenLayer:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find HiddenLayer!" << std::endl;
            return false;
        }
        
        for(UINT i=0; i<numHiddenNeurons; i++){
            UINT tempNeuronID = 0;
            
            file >> word;
            if(word != "HiddenNeuron:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find HiddenNeuron!" << std::endl;
                return false;
            }
            file >> tempNeuronID;
            
            if( tempNeuronID != i+1 ){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find HiddenNeuron ID does not match!" << std::endl;
                return false;
            }
            
            file >> word;
            if(word != "NumInputs:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find NumInputs!" << std::endl;
                return false;
            }
            file >> hiddenLayer[i].numInputs;
            
            //Resize the buffers
            hiddenLayer[i].weights.resize( hiddenLayer[i].numInputs );
            
            file >> word;
            if(word != "Bias:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find Bias!" << std::endl;
                return false;
            }
            file >> hiddenLayer[i].bias;
            
            file >> word;
            if(word != "Gamma:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find Gamma!" << std::endl;
                return false;
            }
            file >> hiddenLayer[i].gamma;
            
            file >> word;
            if(word != "Weights:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find Weights!" << std::endl;
                return false;
            }
            
            for(unsigned int j=0; j<hiddenLayer[i].numInputs; j++){
                file >> hiddenLayer[i].weights[j];
            }
        }
        
        //Load the Output Layer
        file >> word;
        if(word != "OutputLayer:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find OutputLayer!" << std::endl;
            return false;
        }
        
        for(UINT i=0; i<numOutputNeurons; i++){
            UINT tempNeuronID = 0;
            
            file >> word;
            if(word != "OutputNeuron:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find OutputNeuron!" << std::endl;
                return false;
            }
            file >> tempNeuronID;
            
            if( tempNeuronID != i+1 ){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find OuputNeuron ID does not match!!" << std::endl;
                return false;
            }
            
            file >> word;
            if(word != "NumInputs:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find NumInputs!" << std::endl;
                return false;
            }
            file >> outputLayer[i].numInputs;
            
            //Resize the buffers
            outputLayer[i].weights.resize( outputLayer[i].numInputs );
            
            file >> word;
            if(word != "Bias:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find Bias!" << std::endl;
                return false;
            }
            file >> outputLayer[i].bias;
            
            file >> word;
            if(word != "Gamma:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find Gamma!" << std::endl;
                return false;
            }
            file >> outputLayer[i].gamma;
            
            file >> word;
            if(word != "Weights:"){
                file.close();
                errorLog << __GRT_LOG__ << " Failed to find Weights!" << std::endl;
                return false;
            }
            
            for(UINT j=0; j<outputLayer[i].numInputs; j++){
                file >> outputLayer[i].weights[j];
            }
        }
        
    }

    setOutputTargets();
    
    return true;
}

UINT MLP::getNumClasses() const{
    if( classificationModeActive )
    return numOutputNeurons;
    return 0;
}

UINT MLP::getNumInputNeurons() const{
    return numInputNeurons;
}

UINT MLP::getNumHiddenNeurons() const{
    return numHiddenNeurons;
}

UINT MLP::getNumOutputNeurons() const{
    return numOutputNeurons;
}

Neuron::Type MLP::getInputLayerActivationFunction() const{
    return inputLayerActivationFunction;
}

Neuron::Type MLP::getHiddenLayerActivationFunction() const{
    return hiddenLayerActivationFunction;
}

Neuron::Type MLP::getOutputLayerActivationFunction() const{
    return outputLayerActivationFunction;
}

UINT MLP::getNumRandomTrainingIterations() const{
    return numRestarts;
}

Float MLP::getTrainingRate() const{
    return learningRate;
}

Float MLP::getMomentum() const{
    return momentum;
}

Float MLP::getGamma() const{
    return gamma;
}

Float MLP::getTrainingError() const{
    return trainingError;
}

bool MLP::getClassificationModeActive() const{
    return classificationModeActive;
}

bool MLP::getRegressionModeActive() const{
    return !classificationModeActive;
}

Vector< Neuron > MLP::getInputLayer() const{
    return inputLayer;
}

Vector< Neuron > MLP::getHiddenLayer() const{
    return hiddenLayer;
}

Vector< Neuron > MLP::getOutputLayer() const{
    return outputLayer;
}

Vector< VectorFloat > MLP::getTrainingLog() const{
    return trainingErrorLog;
}

bool MLP::getNullRejectionEnabled() const{
    return useNullRejection;
}

Float MLP::getNullRejectionCoeff() const{
    return nullRejectionCoeff;
}

Float MLP::getNullRejectionThreshold() const{
    return nullRejectionThreshold;
}

Float MLP::getMaximumLikelihood() const{
    if( trained ) return maxLikelihood;
    return DEFAULT_NULL_LIKELIHOOD_VALUE;
}

VectorFloat MLP::getClassLikelihoods() const{
    if( trained && classificationModeActive ) return classLikelihoods;
    return VectorFloat();
}

VectorFloat MLP::getClassDistances() const{
    //The class distances is simply the regression data
    if( trained && classificationModeActive ) return regressionData;
    return VectorFloat();
}

UINT MLP::getPredictedClassLabel() const{
    if( trained && classificationModeActive ) return predictedClassLabel;
    return 0;
}

std::string MLP::activationFunctionToString(const Neuron::Type activationFunction) const{
    std::string activationName;
    
    switch(activationFunction){
        case(Neuron::LINEAR):
        activationName = "LINEAR";
        break;
        case(Neuron::SIGMOID):
        activationName = "SIGMOID";
        break;
        case(Neuron::BIPOLAR_SIGMOID):
        activationName = "BIPOLAR_SIGMOID";
        break;
        case(Neuron::TANH):
        activationName = "TANH";
        break;
        default:
        activationName = "UNKNOWN";
        break;
    }
    
    return activationName;
}

Neuron::Type MLP::activationFunctionFromString(const std::string activationName) const{
    Neuron::Type activationFunction = Neuron::LINEAR;
    
    if(activationName == "LINEAR" ){
        activationFunction = Neuron::LINEAR;
        return activationFunction;
    }
    if(activationName == "SIGMOID" ){
        activationFunction = Neuron::SIGMOID;
        return activationFunction;
    }
    if(activationName == "BIPOLAR_SIGMOID" ){
        activationFunction = Neuron::BIPOLAR_SIGMOID;
        return activationFunction;
    }
    if(activationName == "TANH" ){
        activationFunction = Neuron::TANH;
        return activationFunction;
    }
    return activationFunction;
}

bool MLP::validateActivationFunction(const Neuron::Type actvationFunction) const{
    if( actvationFunction >= Neuron::LINEAR && actvationFunction < Neuron::NUMBER_OF_ACTIVATION_FUNCTIONS ) return true;
    return false;
}

bool MLP::setInputLayerActivationFunction(const Neuron::Type activationFunction){
    
    if( !validateActivationFunction(activationFunction) ){
        warningLog << __GRT_LOG__ << " The activation function is not valid. It should be one of the Neuron ActivationFunctions enums." << std::endl;
    }
    
    this->inputLayerActivationFunction = activationFunction;
    
    if( initialized ){
        return init(numInputNeurons,numHiddenNeurons,numOutputNeurons);
    }
    
    return true;
}


bool MLP::setHiddenLayerActivationFunction(const Neuron::Type activationFunction){
    
    if( !validateActivationFunction(activationFunction) ){
        warningLog << __GRT_LOG__ << " The activation function is not valid. It should be one of the Neuron ActivationFunctions enums." << std::endl;
    }
    
    this->hiddenLayerActivationFunction = activationFunction;
    
    if( initialized ){
        return init(numInputNeurons,numHiddenNeurons,numOutputNeurons);
    }
    
    return true;
}


bool MLP::setOutputLayerActivationFunction(const Neuron::Type activationFunction){
    
    if( !validateActivationFunction(activationFunction) ){
        warningLog << __GRT_LOG__ << " The activation function is not valid. It should be one of the Neuron ActivationFunctions enums." << std::endl;
    }
    
    this->outputLayerActivationFunction = activationFunction;
    
    if( initialized ){
        return init(numInputNeurons,numHiddenNeurons,numOutputNeurons);
    }
    
    return true;
}

bool MLP::setTrainingRate(const Float trainingRate){
    return setLearningRate( trainingRate );
}

bool MLP::setMomentum(const Float momentum){
    if( momentum >= 0 && momentum <= 1.0 ){
        this->momentum = momentum;
        return true;
    }
    return false;
}

bool MLP::setGamma(const Float gamma){
    
    if( gamma < 0 ){
        warningLog << __GRT_LOG__ << " Gamma must be greater than zero!" << std::endl;
    }
    
    this->gamma = gamma;
    
    if( initialized ){
        return init(numInputNeurons,numHiddenNeurons,numOutputNeurons);
    }
    
    return true;
}

bool MLP::setNumRandomTrainingIterations(const UINT numRandomTrainingIterations){
    return setNumRestarts(numRandomTrainingIterations);
}

bool MLP::setNullRejection(const bool useNullRejection){
    this->useNullRejection = useNullRejection;
    return true;
}

bool MLP::setNullRejectionCoeff(const Float nullRejectionCoeff){
    if( nullRejectionCoeff > 0 ){
        this->nullRejectionCoeff = nullRejectionCoeff;
        return true;
    }
    return false;
}

bool MLP::loadLegacyModelFromFile( std::fstream &file ){
    
    std::string word;
    
    file >> word;
    if(word != "NumInputNeurons:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find NumInputNeurons!" << std::endl;
        return false;
    }
    file >> numInputNeurons;
    numInputDimensions = numInputNeurons;
    
    file >> word;
    if(word != "NumHiddenNeurons:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find NumHiddenNeurons!" << std::endl;
        return false;
    }
    file >> numHiddenNeurons;
    
    file >> word;
    if(word != "NumOutputNeurons:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find NumOutputNeurons!" << std::endl;
        return false;
    }
    file >> numOutputNeurons;
    
    file >> word;
    if(word != "InputLayerActivationFunction:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find InputLayerActivationFunction!" << std::endl;
        return false;
    }
    file >> word;
    inputLayerActivationFunction = activationFunctionFromString(word);
    
    file >> word;
    if(word != "HiddenLayerActivationFunction:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find HiddenLayerActivationFunction!" << std::endl;
        return false;
    }
    file >> word;
    hiddenLayerActivationFunction = activationFunctionFromString(word);
    
    file >> word;
    if(word != "OutputLayerActivationFunction:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find OutputLayerActivationFunction!" << std::endl;
        return false;
    }
    file >> word;
    outputLayerActivationFunction = activationFunctionFromString(word);
    
    file >> word;
    if(word != "MinNumEpochs:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find MinNumEpochs!" << std::endl;
        return false;
    }
    file >> minNumEpochs;
    
    file >> word;
    if(word != "MaxNumEpochs:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find MaxNumEpochs!" << std::endl;
        return false;
    }
    file >> maxNumEpochs;
    
    file >> word;
    if(word != "NumRandomTrainingIterations:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find NumRandomTrainingIterations!" << std::endl;
        return false;
    }
    file >> numRestarts;
    
    file >> word;
    if(word != "ValidationSetSize:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find ValidationSetSize!" << std::endl;
        return false;
    }
    file >> validationSetSize;
    
    file >> word;
    if(word != "MinChange:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find MinChange!" << std::endl;
        return false;
    }
    file >> minChange;
    
    file >> word;
    if(word != "TrainingRate:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find TrainingRate!" << std::endl;
        return false;
    }
    file >> learningRate;
    
    file >> word;
    if(word != "Momentum:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find Momentum!" << std::endl;
        return false;
    }
    file >> momentum;
    
    file >> word;
    if(word != "Gamma:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find Gamma!" << std::endl;
        return false;
    }
    file >> gamma;
    
    file >> word;
    if(word != "UseValidationSet:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find UseValidationSet!" << std::endl;
        return false;
    }
    file >> useValidationSet;
    
    file >> word;
    if(word != "RandomiseTrainingOrder:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find RandomiseTrainingOrder!" << std::endl;
        return false;
    }
    file >> randomiseTrainingOrder;
    
    file >> word;
    if(word != "UseScaling:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find UseScaling!" << std::endl;
        return false;
    }
    file >> useScaling;
    
    file >> word;
    if(word != "ClassificationMode:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find ClassificationMode!" << std::endl;
        return false;
    }
    file >> classificationModeActive;
    
    file >> word;
    if(word != "UseNullRejection:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find UseNullRejection!" << std::endl;
        return false;
    }
    file >> useNullRejection;
    
    file >> word;
    if(word != "RejectionThreshold:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find RejectionThreshold!" << std::endl;
        return false;
    }
    file >> nullRejectionThreshold;
    
    //Resize the layers
    inputLayer.resize( numInputNeurons );
    hiddenLayer.resize( numHiddenNeurons );
    outputLayer.resize( numOutputNeurons );
    
    //Load the neuron data
    file >> word;
    if(word != "InputLayer:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find InputLayer!" << std::endl;
        return false;
    }
    
    for(UINT i=0; i<numInputNeurons; i++){
        UINT tempNeuronID = 0;
        
        file >> word;
        if(word != "InputNeuron:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find InputNeuron!" << std::endl;
            return false;
        }
        file >> tempNeuronID;
        
        if( tempNeuronID != i+1 ){
            file.close();
            errorLog << __GRT_LOG__ << " InputNeuron ID does not match!" << std::endl;
            return false;
        }
        
        file >> word;
        if(word != "NumInputs:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find NumInputs!" << std::endl;
            return false;
        }
        file >> inputLayer[i].numInputs;
        
        //Resize the buffers
        inputLayer[i].weights.resize( inputLayer[i].numInputs );
        
        file >> word;
        if(word != "Bias:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find Bias!" << std::endl;
            return false;
        }
        file >> inputLayer[i].bias;
        
        file >> word;
        if(word != "Gamma:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find Gamma!" << std::endl;
            return false;
        }
        file >> inputLayer[i].gamma;
        
        file >> word;
        if(word != "Weights:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find Weights!" << std::endl;
            return false;
        }
        
        for(UINT j=0; j<inputLayer[i].numInputs; j++){
            file >> inputLayer[i].weights[j];
        }
    }
    
    //Load the Hidden Layer
    file >> word;
    if(word != "HiddenLayer:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find HiddenLayer!" << std::endl;
        return false;
    }
    
    for(UINT i=0; i<numHiddenNeurons; i++){
        UINT tempNeuronID = 0;
        
        file >> word;
        if(word != "HiddenNeuron:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find HiddenNeuron!" << std::endl;
            return false;
        }
        file >> tempNeuronID;
        
        if( tempNeuronID != i+1 ){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find HiddenNeuron ID does not match!" << std::endl;
            return false;
        }
        
        file >> word;
        if(word != "NumInputs:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find NumInputs!" << std::endl;
            return false;
        }
        file >> hiddenLayer[i].numInputs;
        
        //Resize the buffers
        hiddenLayer[i].weights.resize( hiddenLayer[i].numInputs );
        
        file >> word;
        if(word != "Bias:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find Bias!" << std::endl;
            return false;
        }
        file >> hiddenLayer[i].bias;
        
        file >> word;
        if(word != "Gamma:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find Gamma!" << std::endl;
            return false;
        }
        file >> hiddenLayer[i].gamma;
        
        file >> word;
        if(word != "Weights:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find Weights!" << std::endl;
            return false;
        }
        
        for(unsigned int j=0; j<hiddenLayer[i].numInputs; j++){
            file >> hiddenLayer[i].weights[j];
        }
    }
    
    //Load the Output Layer
    file >> word;
    if(word != "OutputLayer:"){
        file.close();
        errorLog << __GRT_LOG__ << " Failed to find OutputLayer!" << std::endl;
        return false;
    }
    
    for(UINT i=0; i<numOutputNeurons; i++){
        UINT tempNeuronID = 0;
        
        file >> word;
        if(word != "OutputNeuron:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find OutputNeuron!" << std::endl;
            return false;
        }
        file >> tempNeuronID;
        
        if( tempNeuronID != i+1 ){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find OuputNeuron ID does not match!!" << std::endl;
            return false;
        }
        
        file >> word;
        if(word != "NumInputs:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find NumInputs!" << std::endl;
            return false;
        }
        file >> outputLayer[i].numInputs;
        
        //Resize the buffers
        outputLayer[i].weights.resize( outputLayer[i].numInputs );
        
        file >> word;
        if(word != "Bias:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find Bias!" << std::endl;
            return false;
        }
        file >> outputLayer[i].bias;
        
        file >> word;
        if(word != "Gamma:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find Gamma!" << std::endl;
            return false;
        }
        file >> outputLayer[i].gamma;
        
        file >> word;
        if(word != "Weights:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find Weights!" << std::endl;
            return false;
        }
        
        for(UINT j=0; j<outputLayer[i].numInputs; j++){
            file >> outputLayer[i].weights[j];
        }
    }
    
    if( useScaling ){
        //Resize the ranges buffers
        inputVectorRanges.resize( numInputNeurons );
        targetVectorRanges.resize( numOutputNeurons );
        
        //Load the ranges
        file >> word;
        if(word != "InputVectorRanges:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find InputVectorRanges!" << std::endl;
            return false;
        }
        for(UINT j=0; j<inputVectorRanges.size(); j++){
            file >> inputVectorRanges[j].minValue;
            file >> inputVectorRanges[j].maxValue;
        }
        
        file >> word;
        if(word != "OutputVectorRanges:"){
            file.close();
            errorLog << __GRT_LOG__ << " Failed to find OutputVectorRanges!" << std::endl;
            return false;
        }
        for(UINT j=0; j<targetVectorRanges.size(); j++){
            file >> targetVectorRanges[j].minValue;
            file >> targetVectorRanges[j].maxValue;
        }
    }
    
    initialized = true;
    trained = true;
    
    return true;
}

bool MLP::setOutputTargets(){

    switch( outputLayerActivationFunction ){
        case Neuron::SIGMOID:
            outputTargets.minValue = 0.0;
            outputTargets.maxValue = 1.0;
        break;
        case Neuron::TANH:
            outputTargets.minValue = -1.0;
            outputTargets.maxValue = 1.0;
        break;
        default:
            outputTargets.minValue = 0;
            outputTargets.maxValue = 1.0;
        break;
    }

    return true;
}

GRT_END_NAMESPACE