SVM.cpp

/*
 GRT MIT License
 Copyright (c) <2012> <Nicholas Gillian, Media Lab, MIT>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software 
 and associated documentation files (the "Software"), to deal in the Software without restriction, 
 including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
 and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, 
 subject to the following conditions:
 
 The above copyright notice and this permission notice shall be included in all copies or substantial 
 portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT 
 LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#include "SVM.h"

GRT_BEGIN_NAMESPACE
 
//Register the SVM module with the Classifier base class
RegisterClassifierModule< SVM > SVM::registerModule("SVM");

SVM::SVM(UINT kernelType,UINT svmType,bool useScaling,bool useNullRejection,bool useAutoGamma,Float gamma,UINT degree,Float coef0,Float nu,Float C,bool useCrossValidation,UINT kFoldValue){
 
 //Setup the default SVM parameters
 model = NULL;
 param.weight_label = NULL;
 param.weight = NULL;
 prob.l = 0;
 prob.x = NULL;
 prob.y = NULL;
 trained = false;
 problemSet = false;
 param.svm_type = C_SVC;
 param.kernel_type = LINEAR_KERNEL;
 param.degree = 3;
 param.gamma = 0;
 param.coef0 = 0;
 param.nu = 0.5;
 param.cache_size = 100;
 param.C = 1;
 param.eps = 1e-3;
 param.p = 0.1;
 param.shrinking = 1;
 param.probability = 1;
 param.nr_weight = 0;
 param.weight_label = NULL;
 param.weight = NULL;
 this->useScaling = false;
 this->useCrossValidation = false;
 this->useNullRejection = false;
 this->useAutoGamma = true;
 classificationThreshold = 0.5;
 crossValidationResult = 0;
 
 classType = "SVM";
 classifierType = classType;
 classifierMode = STANDARD_CLASSIFIER_MODE;
 debugLog.setProceedingText("[DEBUG SVM]");
 errorLog.setProceedingText("[ERROR SVM]");
 trainingLog.setProceedingText("[TRAINING SVM]");
 warningLog.setProceedingText("[WARNING SVM]");
 
 init(kernelType,svmType,useScaling,useNullRejection,useAutoGamma,gamma,degree,coef0,nu,C,useCrossValidation,kFoldValue);
}
 
SVM::SVM(const SVM &rhs){
 model = NULL;
 param.weight_label = NULL;
 param.weight = NULL;
 prob.l = 0;
 prob.x = NULL;
 prob.y = NULL;
 classType = "SVM";
 classifierType = classType;
 classifierMode = STANDARD_CLASSIFIER_MODE;
 debugLog.setProceedingText("[DEBUG SVM]");
 errorLog.setProceedingText("[ERROR SVM]");
 trainingLog.setProceedingText("[TRAINING SVM]");
 warningLog.setProceedingText("[WARNING SVM]");
 *this = rhs;
}


SVM::~SVM(){
 clear();
}
 
SVM& SVM::operator=(const SVM &rhs){
 if( this != &rhs ){
 
 this->clear();
 
 //SVM variables
 this->problemSet = false; //We do not copy the problem set
 this->model = rhs.deepCopyModel();
 this->deepCopyParam( rhs.param, this->param );
 this->numInputDimensions = rhs.numInputDimensions;
 this->kFoldValue = rhs.kFoldValue;
 this->classificationThreshold = rhs.classificationThreshold;
 this->crossValidationResult = rhs.crossValidationResult;
 this->useAutoGamma = rhs.useAutoGamma;
 this->useCrossValidation = rhs.useCrossValidation;
 
 //Classifier variables
 copyBaseVariables( (Classifier*)&rhs );
 }
 return *this;
}
 
bool SVM::deepCopyFrom(const Classifier *classifier){
 
 if( classifier == NULL ) return false;
 
 if( this->getClassifierType() == classifier->getClassifierType() ){
 SVM *ptr = (SVM*)classifier;
 
 this->clear();
 
 //SVM variables
 this->problemSet = false;
 this->model = ptr->deepCopyModel();
 this->deepCopyParam( ptr->param, this->param );
 this->numInputDimensions = ptr->numInputDimensions;
 this->kFoldValue = ptr->kFoldValue;
 this->classificationThreshold = ptr->classificationThreshold;
 this->crossValidationResult = ptr->crossValidationResult;
 this->useAutoGamma = ptr->useAutoGamma;
 this->useCrossValidation = ptr->useCrossValidation;
 
 //Classifier variables
 return copyBaseVariables( classifier );
 }
 
 return false;
}
 
bool SVM::train_(ClassificationData &trainingData){
 
 //Clear any previous model
 clear();
 
 if( trainingData.getNumSamples() == 0 ){
 errorLog << "train_(ClassificationData &trainingData) - Training data has zero samples!" << std::endl;
 return false;
 }
 
 //Convert the labelled classification data into the LIBSVM data format
 if( !convertClassificationDataToLIBSVMFormat(trainingData) ){
 errorLog << "train_(ClassificationData &trainingData) - Failed To Convert Labelled Classification Data To LIBSVM Format!" << std::endl;
 return false;
 }
 
 if( useAutoGamma ) param.gamma = 1.0/numInputDimensions;
 
 //Train the model
 bool trainingResult = trainSVM();
 
 if(! trainingResult ){
 errorLog << "train_(ClassificationData &trainingData) - Failed To Train SVM Model!" << std::endl;
 return false;
 }
 
 return true;
}

bool SVM::predict_(VectorFloat &inputVector){
 
 if( !trained ){
 errorLog << "predict_(VectorFloat &inputVector) - The SVM model has not been trained!" << std::endl;
 return false;
 }
 
 if( inputVector.size() != numInputDimensions ){
 errorLog << "predict_(VectorFloat &inputVector) - The size of the input vector (" << inputVector.size() << ") does not match the number of features of the model (" << numInputDimensions << ")" << std::endl;
 return false;
 }
 
 if( param.probability == 1 ){
 if( !predictSVM( inputVector, maxLikelihood, classLikelihoods ) ){
 errorLog << "predict(VectorFloat inputVector) - Prediction Failed!" << std::endl;
 return false;
 }
 }else{
 if( !predictSVM( inputVector ) ){
 errorLog << "predict(VectorFloat inputVector) - Prediction Failed!" << std::endl;
 return false;
 }
 }
 
 return true;
}
 
bool SVM::init(UINT kernelType,UINT svmType,bool useScaling,bool useNullRejection,bool useAutoGamma,Float gamma,UINT degree,Float coef0,Float nu,Float C,bool useCrossValidation,UINT kFoldValue){
 
 //Clear any previous models or problems
 clear();
 
 //Validate the kernerlType
 if( !validateKernelType(kernelType) ){
 errorLog << "init(...) - Unknown kernelType!\n";
 return false;
 }
 
 if( !validateSVMType(svmType) ){
 errorLog << "init(...) - Unknown kernelType!\n";
 return false;
 }
 
 param.svm_type = (int)svmType;
 param.kernel_type = (int)kernelType;
 param.degree = (int)degree;
 param.gamma = gamma;
 param.coef0 = coef0;
 param.nu = nu;
 param.cache_size = 100;
 param.C = C;
 param.eps = 1e-3;
 param.p = 0.1;
 param.shrinking = 1;
 param.probability = 1;
 param.nr_weight = 0;
 param.weight_label = NULL;
 param.weight = NULL;
 this->useScaling = useScaling;
 this->useCrossValidation = useCrossValidation;
 this->useNullRejection = useNullRejection;
 this->useAutoGamma = useAutoGamma;
 classificationThreshold = 0.5;
 crossValidationResult = 0;

 return true;
}
 
void SVM::deleteProblemSet(){
 if( problemSet ){
 for(int i=0; i<prob.l; i++){
 delete[] prob.x[i];
 prob.x[i] = NULL;
 }
 delete[] prob.x;
 delete[] prob.y;
 prob.l = 0;
 prob.x = NULL;
 prob.y = NULL;
 problemSet = false;
 }
}

void SVM::initDefaultSVMSettings(){

 //Clear any previous models, parameters or probelms
 clear();

 //Setup the SVM parameters
 param.svm_type = C_SVC;
 param.kernel_type = LINEAR_KERNEL;
 param.degree = 3;
 param.gamma = 0;
 param.coef0 = 0;
 param.nu = 0.5;
 param.cache_size = 100;
 param.C = 1;
 param.eps = 1e-3;
 param.p = 0.1;
 param.shrinking = 1;
 param.probability = 1;
 param.nr_weight = 0;
 param.weight_label = NULL;
 param.weight = NULL;
 useCrossValidation = false;
 kFoldValue = 10;
 useAutoGamma = true;
}

bool SVM::validateProblemAndParameters(){
 //Check the parameters match the problem
 const char *errorMsg = svm_check_parameter(&prob,&param);

 if( errorMsg ){
 errorLog << "validateProblemAndParameters() - Parameters do not match problem!" << std::endl;
 return false;
 }

 return true;
}

bool SVM::trainSVM(){

 crossValidationResult = 0;

 //Erase any previous models
 if( trained ){
 svm_free_and_destroy_model(&model);
 trained = false;
 }

 //Check to make sure the problem has been set
 if( !problemSet ){
 errorLog << "trainSVM() - Problem not set!" << std::endl;
 return false;
 }
 
 //Verify the problem and the parameters
 if( !validateProblemAndParameters() ) return false;

 //Scale the training data if needed
 if( useScaling ){
 for(int i=0; i<prob.l; i++)
 for(UINT j=0; j<numInputDimensions; j++)
 prob.x[i][j].value = grt_scale(prob.x[i][j].value,ranges[j].minValue,ranges[j].maxValue,SVM_MIN_SCALE_RANGE,SVM_MAX_SCALE_RANGE);
 }

 if( useCrossValidation ){
 int i;
 Float total_correct = 0;
 Float total_error = 0;
 Float sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0;
 Float *target = new Float[prob.l];

 svm_cross_validation(&prob,&param,kFoldValue,target);
 if( param.svm_type == EPSILON_SVR || param.svm_type == NU_SVR )
 {
 for(i=0;i<prob.l;i++)
 {
 Float y = prob.y[i];
 Float v = target[i];
 total_error += (v-y)*(v-y);
 sumv += v;
 sumy += y;
 sumvv += v*v;
 sumyy += y*y;
 sumvy += v*y;
 }
 crossValidationResult = total_error/prob.l;
 }
 else
 {
 for(i=0;i<prob.l;i++){
 if(target[i] == prob.y[i]){
 ++total_correct;
 }
 }
 crossValidationResult = total_correct/prob.l*100.0;
 }
 delete[] target;
 }
 
 //Train the SVM - if we are running cross validation then the CV will be run first followed by a full train
 model = svm_train(&prob,&param);

 if( model == NULL ){
 errorLog << "trainSVM() - Failed to train SVM Model!" << std::endl;
 return false;
 }

 if( model != NULL ){
 trained = true;
 numClasses = getNumClasses();
 classLabels.resize( getNumClasses() );
 for(UINT k=0; k<getNumClasses(); k++){
 classLabels[k] = model->label[k];
 }
 classLikelihoods.resize(numClasses,DEFAULT_NULL_LIKELIHOOD_VALUE);
 classDistances.resize(numClasses,DEFAULT_NULL_DISTANCE_VALUE);
 }

 return trained;
}
 
bool SVM::predictSVM(VectorFloat &inputVector){

 if( !trained || inputVector.size() != numInputDimensions ) return false;

 svm_node *x = NULL;

 //Copy the input data into the SVM format
 x = new svm_node[numInputDimensions+1];
 for(UINT j=0; j<numInputDimensions; j++){
 x[j].index = (int)j+1;
 x[j].value = inputVector[j];
 }
 //The last value in the input vector must be set to -1
 x[numInputDimensions].index = -1;
 x[numInputDimensions].value = 0;

 //Scale the input data if required
 if( useScaling ){
 for(UINT i=0; i<numInputDimensions; i++)
 x[i].value = grt_scale(x[i].value,ranges[i].minValue,ranges[i].maxValue,SVM_MIN_SCALE_RANGE,SVM_MAX_SCALE_RANGE);
 }

 //Perform the SVM prediction
 Float predict_label = svm_predict(model,x);

 //We can't do null rejection without the probabilities, so just set the predicted class
 predictedClassLabel = (UINT)predict_label;

 //Clean up the memory
 delete[] x;

 return true;
}

bool SVM::predictSVM(VectorFloat &inputVector,Float &maxProbability, VectorFloat &probabilites){

 if( !trained || param.probability == 0 || inputVector.size() != numInputDimensions ) return false;

 Float *prob_estimates = NULL;
 svm_node *x = NULL;

 //Setup the memory for the probability estimates
 prob_estimates = new Float[ model->nr_class ];

 //Copy the input data into the SVM format
 x = new svm_node[numInputDimensions+1];
 for(UINT j=0; j<numInputDimensions; j++){
 x[j].index = (int)j+1;
 x[j].value = inputVector[j];
 }
 //The last value in the input vector must be set to -1
 x[numInputDimensions].index = -1;
 x[numInputDimensions].value = 0;

 //Scale the input data if required
 if( useScaling ){
 for(UINT j=0; j<numInputDimensions; j++)
 x[j].value = grt_scale(x[j].value,ranges[j].minValue,ranges[j].maxValue,SVM_MIN_SCALE_RANGE,SVM_MAX_SCALE_RANGE);
 }

 //Perform the SVM prediction
 Float predict_label = svm_predict_probability(model,x,prob_estimates);

 predictedClassLabel = 0;
 maxProbability = 0;
 probabilites.resize(model->nr_class);
 for(int k=0; k<model->nr_class; k++){
 if( maxProbability < prob_estimates[k] ){
 maxProbability = prob_estimates[k];
 predictedClassLabel = k+1;
 maxLikelihood = maxProbability;
 }
 probabilites[k] = prob_estimates[k];
 }

 if( !useNullRejection ) predictedClassLabel = (UINT)predict_label;
 else{
 if( maxProbability >= classificationThreshold ){
 predictedClassLabel = (UINT)predict_label;
 }else predictedClassLabel = GRT_DEFAULT_NULL_CLASS_LABEL;
 }

 //Clean up the memory
 delete[] prob_estimates;
 delete[] x;

 return true;
}
 
bool SVM::convertClassificationDataToLIBSVMFormat(ClassificationData &trainingData){
 
 //clear any previous problems
 deleteProblemSet();
 
 const UINT numTrainingExamples = trainingData.getNumSamples();
 numInputDimensions = trainingData.getNumDimensions();
 
 //Compute the ranges encase the data should be scaled
 ranges = trainingData.getRanges();
 
 //Init the memory
 prob.l = numTrainingExamples;
 prob.x = new svm_node*[numTrainingExamples];
 prob.y = new Float[numTrainingExamples];
 problemSet = true;
 
 for(UINT i=0; i<numTrainingExamples; i++){
 //Set the class ID
 prob.y[i] = trainingData[i].getClassLabel();
 
 //Assign the memory for this training example, note that a dummy node is needed at the end of the vector
 prob.x[i] = new svm_node[numInputDimensions+1];
 for(UINT j=0; j<numInputDimensions; j++){
 prob.x[i][j].index = j+1;
 prob.x[i][j].value = trainingData[i].getSample()[j];
 }
 prob.x[i][numInputDimensions].index = -1; //Assign the final node value
 prob.x[i][numInputDimensions].value = 0;
 }
 
 return true;
}

bool SVM::saveModelToFile( std::fstream &file ) const{
 
 if( !file.is_open() ){
 return false;
 }
 
 file << "SVM_MODEL_FILE_V2.0\n";
 
 //Write the classifier settings to the file
 if( !Classifier::saveBaseSettingsToFile(file) ){
 errorLog <<"saveModelToFile(fstream &file) - Failed to save classifier base settings to file!" << std::endl;
 return false;
 }
 
 const svm_parameter& param = trained ? model->param : this->param;
 
 file << "ModelType: ";
 switch( param.svm_type ){
 case(C_SVC):
 file << "C_SVC";
 break;
 case(NU_SVC):
 file << "NU_SVC";
 break;
 case(ONE_CLASS):
 file << "ONE_CLASS";
 break;
 case(EPSILON_SVR):
 file << "EPSILON_SVR";
 break;
 case(NU_SVR):
 file << "NU_SVR";
 break;
 default:
 errorLog << "saveModelToFile(fstream &file) - Invalid model type: " << param.svm_type << std::endl;
 return false;
 break;
 }
 file << std::endl;
 
 file << "KernelType: ";
 switch(param.kernel_type){
 case(LINEAR):
 file << "LINEAR";
 break;
 case(POLY):
 file << "POLYNOMIAL";
 break;
 case(RBF):
 file << "RBF";
 break;
 case(SIGMOID):
 file << "SIGMOID";
 break;
 case(PRECOMPUTED):
 file << "PRECOMPUTED";
 break;
 default:
 errorLog << "saveModelToFile(fstream &file) - Invalid kernel type: " << param.kernel_type << std::endl;
 return false;
 break;
 }
 file << std::endl;
 file << "Degree: " << param.degree << std::endl;
 file << "Gamma: " << param.gamma << std::endl;
 file << "Coef0: " << param.coef0 << std::endl;
 file << "NumberOfFeatures: " << numInputDimensions << std::endl;
 file << "UseShrinking: " << param.shrinking << std::endl;
 file << "UseProbability: " << param.probability << std::endl;
 
 if( trained ){
 UINT numClasses = (UINT)model->nr_class;
 UINT numSV = (UINT)model->l;
 file << "NumberOfSupportVectors: " << numSV << std::endl;
 
 file << "RHO: \n";
 for(UINT i=0;i<numClasses*(numClasses-1)/2;i++) file << model->rho[i] << "\t";
 file << "\n";
 
 if(model->label){
 file << "Label: \n";
 for(UINT i=0;i<numClasses;i++) file << model->label[i] << "\t";
 file << "\n";
 }
 
 if(model->probA){ // regression has probA only
 file << "ProbA: \n";
 for(UINT i=0;i<numClasses*(numClasses-1)/2;i++) file << model->probA[i] << "\t";
 file << "\n";
 }
 
 if(model->probB){
 file << "ProbB: \n";
 for(UINT i=0;i<numClasses*(numClasses-1)/2;i++) file << model->probB[i] << "\t";
 file << "\n";
 }
 
 if(model->nSV){
 file << "NumSupportVectorsPerClass: \n";
 for(UINT i=0;i<numClasses;i++) file << model->nSV[i] << "\t";
 file << "\n";
 }
 
 file << "SupportVectors: \n";
 
 const Float * const *sv_coef = model->sv_coef;
 const svm_node * const *SV = model->SV;
 
 for(UINT i=0;i<numSV;i++){
 for(UINT j=0;j<numClasses-1;j++)
 file << sv_coef[j][i] << "\t";
 
 const svm_node *p = SV[i];
 
 if(param.kernel_type == PRECOMPUTED) file << (int) p->value << "\t";
 else{
 while(p->index != -1){
 file << p->index << "\t" << p->value << "\t";
 p++;
 }
 file << "\n";
 }
 }
 }
 
 return true;
}
 
bool SVM::loadModelFromFile( std::fstream &file ){
 
 std::string word;
 UINT numSV = 0;
 UINT halfNumClasses = 0;
 
 //Clear any previous models, parameters or problems
 clear();
 
 if( !file.is_open() ){
 errorLog << "loadModelFromFile(fstream &file) - The file is not open!" << std::endl;
 return false;
 }
 
 //Read the file header
 file >> word;
 
 //Check to see if we should load a legacy file
 if( word == "SVM_MODEL_FILE_V1.0" ){
 return loadLegacyModelFromFile( file );
 }
 
 //Check to make sure this is a file with the correct File Format
 if( word != "SVM_MODEL_FILE_V2.0" ){
 errorLog << "loadModelFromFile(fstream &file) - Invalid file format!" << std::endl;
 clear();
 return false;
 }

 //Load the base settings from the file
 if( !Classifier::loadBaseSettingsFromFile(file) ){
 errorLog << "loadModelFromFile(string filename) - Failed to load base settings from file!" << std::endl;
 return false;
 }
 
 //Init the memory for the model
 model = new svm_model;
 model->nr_class = 0;
 model->l = 0;
 model->SV = NULL;
 model->sv_coef = NULL;
 model->rho = NULL;
 model->probA = NULL;
 model->probB = NULL;
 model->label = NULL;
 model->nSV = NULL;
 model->label = NULL;
 model->nSV = NULL;
 model->free_sv = 0; //This will be set to 1 if everything is loaded OK
 
 //Init the memory for the parameters
 model->param.svm_type = 0;
 model->param.kernel_type = 0;
 model->param.degree = 0;
 model->param.gamma = 0;
 model->param.coef0 = 0;
 model->param.cache_size = 0;
 model->param.eps = 0;
 model->param.C = 0;
 model->param.nr_weight = 0;
 model->param.weight_label = NULL;
 model->param.weight = NULL;
 model->param.nu = 0;
 model->param.p = 0;
 model->param.shrinking = 0;
 model->param.probability = 1;

 //Load the model type
 file >> word;
 if(word != "ModelType:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find ModelType header!" << std::endl;
 clear();
 return false;
 }
 file >> word;
 if( word == "C_SVC" ){
 model->param.svm_type = C_SVC;
 }else{
 if( word == "NU_SVC" ){
 model->param.svm_type = NU_SVC;
 }else{
 if( word == "ONE_CLASS" ){
 model->param.svm_type = ONE_CLASS; 
 }else{
 if( word == "EPSILON_SVR" ){
 model->param.svm_type = EPSILON_SVR;
 }else{
 if( word == "NU_SVR" ){
 model->param.svm_type = NU_SVR;
 }else{
 errorLog << "loadModelFromFile(fstream &file) - Failed to find SVM type!" << std::endl;
 clear();
 return false;
 }
 }
 }
 }
 }
 
 //Load the model type
 file >> word;
 if(word != "KernelType:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find kernel type!" << std::endl;
 clear();
 return false;
 }
 file >> word;
 if( word == "LINEAR" ){
 model->param.kernel_type = LINEAR;
 }else{
 if( word == "POLYNOMIAL" ){
 model->param.kernel_type = POLY;
 }else{
 if( word == "RBF" ){
 model->param.kernel_type = RBF; 
 }else{
 if( word == "SIGMOID" ){
 model->param.kernel_type = SIGMOID;
 }else{
 if( word == "PRECOMPUTED" ){
 model->param.kernel_type = PRECOMPUTED;
 }else{
 errorLog << "loadModelFromFile(fstream &file) - Failed to find kernel type!" << std::endl;
 clear();
 return false;
 }
 }
 }
 }
 }
 
 //Load the degree
 file >> word;
 if(word != "Degree:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find Degree header!" << std::endl;
 clear();
 return false;
 }
 file >> model->param.degree;
 
 //Load the gamma
 file >> word;
 if(word != "Gamma:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find Gamma header!" << std::endl;
 clear();
 return false;
 }
 file >> model->param.gamma;
 
 //Load the Coef0
 file >> word;
 if(word != "Coef0:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find Coef0 header!" << std::endl;
 clear();
 return false;
 }
 file >> model->param.coef0;
 
 //Load the NumberOfFeatures
 file >> word;
 if(word != "NumberOfFeatures:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find NumberOfFeatures header!" << std::endl;
 clear();
 return false;
 }
 file >> numInputDimensions;
 
 //Load the UseShrinking
 file >> word;
 if(word != "UseShrinking:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find UseShrinking header!" << std::endl;
 clear();
 return false;
 }
 file >> model->param.shrinking;
 
 //Load the UseProbability
 file >> word;
 if(word != "UseProbability:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find UseProbability header!" << std::endl;
 clear();
 return false;
 }
 file >> model->param.probability;
 
 if( trained ){
 //Load the NumberOfSupportVectors
 file >> word;
 if(word != "NumberOfSupportVectors:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find NumberOfSupportVectors header!" << std::endl;
 clear();
 return false;
 }
 file >> numSV;
 
 //Setup the values
 halfNumClasses = numClasses*(numClasses-1)/2;
 model->nr_class = numClasses;
 model->l = numSV;
 
 //Load the RHO
 file >> word;
 if(word != "RHO:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find RHO header!" << std::endl;
 clear();
 return false;
 }
 model->rho = new Float[ halfNumClasses ];
 for(UINT i=0;i<numClasses*(numClasses-1)/2;i++) file >> model->rho[i];
 
 //See if we can load the Labels
 file >> word;
 if(word != "Label:"){
 model->label = NULL;
 }else{
 model->label = new int[ numClasses ];
 for(UINT i=0;i<numClasses;i++) file >> model->label[i];
 //We only need to read a new line if we found the label!
 file >> word;
 }
 
 //See if we can load the ProbA
 //We don't need to read another line here
 if(word != "ProbA:"){
 model->probA = NULL;
 }else{
 model->probA = new Float[ halfNumClasses ];
 for(UINT i=0;i<numClasses*(numClasses-1)/2;i++) file >> model->probA[i];
 //We only need to read a new line if we found the label!
 file >> word;
 }
 
 //See if we can load the ProbB
 //We don't need to read another line here
 if(word != "ProbB:"){
 model->probB = NULL;
 }else{
 model->probB = new Float[ halfNumClasses ];
 for(UINT i=0;i<numClasses*(numClasses-1)/2;i++) file >> model->probB[i];
 //We only need to read a new line if we found the label!
 file >> word;
 }
 
 //See if we can load the NumSupportVectorsPerClass
 //We don't need to read another line here
 if( word == "NumSupportVectorsPerClass:" ){
 model->nSV = new int[ numClasses ];
 for(UINT i=0; i<numClasses; i++) file >> model->nSV[i];
 //We only need to read a new line if we found the label!
 file >> word;
 }else{
 model->nSV = NULL;
 }
 
 //Load the SupportVectors
 //We don't need to read another line here
 if(word != "SupportVectors:"){
 errorLog << "loadModelFromFile(fstream &file) - Failed to find SupportVectors header!" << std::endl;
 clear();
 return false;
 }
 
 //Setup the memory
 model->sv_coef = new Float*[numClasses-1];
 for(UINT j=0;j<numClasses-1;j++) model->sv_coef[j] = new Float[numSV];
 model->SV = new svm_node*[numSV];
 
 for(UINT i=0; i<numSV; i++){
 for(UINT j=0; j<numClasses-1; j++){
 file >> model->sv_coef[j][i];
 }
 
 model->SV[i] = new svm_node[numInputDimensions+1];
 
 if(model->param.kernel_type == PRECOMPUTED) file >> model->SV[i][0].value;
 else{
 for(UINT j=0; j<numInputDimensions; j++){
 file >> model->SV[i][j].index;
 file >> model->SV[i][j].value;
 }
 model->SV[i][numInputDimensions].index = -1; //Assign the final node value
 model->SV[i][numInputDimensions].value = 0;
 }
 }
 
 //Set the class labels
 this->numClasses = getNumClasses();
 classLabels.resize(getNumClasses());
 for(UINT k=0; k<getNumClasses(); k++){
 classLabels[k] = model->label[k];
 }
 
 //The SV have now been loaded so flag that they should be deleted
 model->free_sv = 1;
 
 //Resize the prediction results to make sure it is setup for realtime prediction
 maxLikelihood = DEFAULT_NULL_LIKELIHOOD_VALUE;
 bestDistance = DEFAULT_NULL_DISTANCE_VALUE;
 classLikelihoods.resize(numClasses,DEFAULT_NULL_LIKELIHOOD_VALUE);
 classDistances.resize(numClasses,DEFAULT_NULL_DISTANCE_VALUE);
 }
 
 return true;
}
 
bool SVM::clear(){
 
 //Clear the base class
 Classifier::clear();
 
 crossValidationResult = 0;
 trained = false;
 svm_free_and_destroy_model(&model);
 svm_destroy_param(&param);
 deleteProblemSet();
 
 return true;
}
 
bool SVM::getIsCrossValidationTrainingEnabled() const{
 return useCrossValidation;
}
 
bool SVM::getIsAutoGammaEnabled() const{
 return useAutoGamma;
}
 
std::string SVM::getSVMType() const{

 const struct svm_parameter *paramPtr = NULL;
 std::string modelName = "UNKNOWN";
 if( trained ){
 paramPtr = &model->param;
 }else paramPtr = &param;

 switch(paramPtr->svm_type){
 case(C_SVC):
 modelName = "C_SVC";
 break;
 case(NU_SVC):
 modelName = "NU_SVC";
 break;
 case(ONE_CLASS):
 modelName = "ONE_CLASS";
 break;
 case(EPSILON_SVR):
 modelName = "EPSILON_SVR";
 break;
 case(NU_SVR):
 modelName = "NU_SVR";
 break;
 default:
 break;
 }

 return modelName;
}
 
std::string SVM::getKernelType() const{
 const struct svm_parameter *paramPtr = NULL;
 std::string modelName = "UNKNOWN";
 if( trained ){
 paramPtr = &model->param;
 }else paramPtr = &param;
 
 switch(paramPtr->kernel_type){
 case(LINEAR_KERNEL):
 modelName = "LINEAR_KERNEL";
 break;
 case(POLY_KERNEL):
 modelName = "POLY_KERNEL";
 break;
 case(RBF_KERNEL):
 modelName = "RBF_KERNEL";
 break;
 case(SIGMOID_KERNEL):
 modelName = "SIGMOID_KERNEL";
 break;
 case(PRECOMPUTED_KERNEL):
 modelName = "PRECOMPUTED_KERNEL";
 break;
 default:
 break;
 }
 return modelName;
}
 
UINT SVM::getNumClasses() const{
 if( !trained ) return 0;
 return (UINT) model->nr_class;
}
 
UINT SVM::getDegree() const{
 if( trained ){
 return (UINT)model->param.degree;
 }
 return (UINT)param.gamma;
}
 
Float SVM::getGamma() const{
 if( trained ){
 return model->param.gamma;
 }
 return param.gamma;
}
 
Float SVM::getNu() const{
 if( trained ){
 return model->param.nu;
 }
 return param.gamma;
}

Float SVM::getCoef0() const{
 if( trained ){
 return model->param.coef0;
 }
 return param.gamma;
}

Float SVM::getC() const{
 if( trained ){
 return model->param.C;
 }
 return param.gamma;
}
 
Float SVM::getCrossValidationResult() const{ return crossValidationResult; }

bool SVM::setSVMType(const UINT svmType){
 if( validateSVMType(svmType) ){
 param.svm_type = (int)svmType;
 return true;
 }
 return false;
}
 
bool SVM::setKernelType(const UINT kernelType){
 if( validateKernelType(kernelType) ){
 param.kernel_type = (int)kernelType;
 return true;
 }
 warningLog << "setKernelType(UINT kernelType) - Failed to set kernel type, unknown kernelType!" << std::endl;
 return false;
}
 
bool SVM::setGamma(const Float gamma){
 if( !useAutoGamma ){
 this->param.gamma = gamma;
 return true;
 }
 warningLog << "setGamma(Float gamma) - Failed to set gamma, useAutoGamma is enabled, setUseAutoGamma to false first!" << std::endl;
 return false;
}
 
bool SVM::setDegree(const UINT degree){
 this->param.degree = (int)degree;
 return true;
}
 
bool SVM::setNu(const Float nu){
 this->param.nu = nu;
 return true;
}

bool SVM::setCoef0(const Float coef0){
 this->param.coef0 = coef0;
 return true;
}

bool SVM::setC(const Float C){
 this->param.C = C;
 return true;
}
 
bool SVM::setKFoldCrossValidationValue(const UINT kFoldValue){
 if( kFoldValue > 0 ){
 this->kFoldValue = kFoldValue;
 return true;
 }
 warningLog << "setKFoldCrossValidationValue(const UINT kFoldValue) - Failed to set kFoldValue, the kFoldValue must be greater than 0!" << std::endl;
 return false;
}
 
bool SVM::enableAutoGamma(const bool useAutoGamma){
 this->useAutoGamma = useAutoGamma;
 return true;
}

bool SVM::enableCrossValidationTraining(const bool useCrossValidation){
 this->useCrossValidation = useCrossValidation;
 return true;
}
 
bool SVM::validateSVMType(const UINT svmType){
 if( svmType == C_SVC ){
 return true;
 }
 if( svmType == NU_SVC ){
 return true;
 }
 if( svmType == ONE_CLASS ){
 return true;
 }
 if( svmType == EPSILON_SVR ){
 return true;
 }
 if( svmType == NU_SVR ){
 return true;
 }
 return false;
}
 
bool SVM::validateKernelType(const UINT kernelType){
 if( kernelType == LINEAR_KERNEL ){
 return true;
 }
 if( kernelType == POLY_KERNEL ){
 return true;
 }
 if( kernelType == RBF_KERNEL ){
 return true;
 }
 if( kernelType == SIGMOID_KERNEL ){
 return true;
 }
 if( kernelType == PRECOMPUTED_KERNEL ){
 return true;
 }
 return false;
}
 
struct svm_model* SVM::deepCopyModel() const{
 
 if( model == NULL ) return NULL;
 
 UINT halfNumClasses = 0;
 
 //Init the memory for the model
 struct svm_model *m = new svm_model;
 m->nr_class = 0;
 m->l = 0;
 m->SV = NULL;
 m->sv_coef = NULL;
 m->rho = NULL;
 m->probA = NULL;
 m->probB = NULL;
 m->label = NULL;
 m->nSV = NULL;
 m->label = NULL;
 m->nSV = NULL;
 m->free_sv = 0; //This will be set to 1 if everything is loaded OK
 
 //Init the memory for the parameters
 m->param.svm_type = 0;
 m->param.kernel_type = 0;
 m->param.degree = 0;
 m->param.gamma = 0;
 m->param.coef0 = 0;
 m->param.cache_size = 0;
 m->param.eps = 0;
 m->param.C = 0;
 m->param.nr_weight = 0;
 m->param.weight_label = NULL;
 m->param.weight = NULL;
 m->param.nu = 0;
 m->param.p = 0;
 m->param.shrinking = 0;
 m->param.probability = 1;
 
 //Copy the parameters
 m->param.svm_type = model->param.svm_type;
 m->param.kernel_type = model->param.kernel_type ;
 m->param.degree = model->param.degree;
 m->param.gamma = model->param.gamma;
 m->param.coef0 = model->param.coef0;
 m->nr_class = model->nr_class;
 m->l = model->l;
 m->param.shrinking = model->param.shrinking;
 m->param.probability = model->param.probability;
 
 //Setup the values
 halfNumClasses = model->nr_class*(model->nr_class-1)/2;

 m->rho = new Float[ halfNumClasses ];
 for(int i=0;i <model->nr_class*(model->nr_class-1)/2; i++) m->rho[i] = model->rho[i];
 
 if( model->label != NULL ){
 m->label = new int[ model->nr_class ];
 for(int i=0;i<model->nr_class;i++) m->label[i] = model->label[i];
 }
 
 if( model->probA != NULL ){
 m->probA = new Float[ halfNumClasses ];
 for(UINT i=0;i<halfNumClasses; i++) m->probA[i] = model->probA[i];
 }
 
 if( model->probB != NULL ){
 m->probB = new Float[ halfNumClasses ];
 for(UINT i=0; i<halfNumClasses; i++) m->probB[i] = model->probB[i];
 }
 
 if( model->nSV != NULL ){
 m->nSV = new int[ model->nr_class ];
 for(int i=0; i<model->nr_class; i++) m->nSV[i] = model->nSV[i];
 }
 
 //Setup the memory
 m->sv_coef = new Float*[numClasses-1];
 for(UINT j=0;j<numClasses-1;j++) m->sv_coef[j] = new Float[model->l];
 m->SV = new svm_node*[model->l];
 
 for(int i=0; i<model->l; i++){
 for(int j=0; j<model->nr_class-1; j++){
 m->sv_coef[j][i] = model->sv_coef[j][i];
 }
 
 m->SV[i] = new svm_node[numInputDimensions+1];
 
 if(model->param.kernel_type == PRECOMPUTED) m->SV[i][0].value = model->SV[i][0].value;
 else{
 for(UINT j=0; j<numInputDimensions; j++){
 m->SV[i][j].index = model->SV[i][j].index;
 m->SV[i][j].value = model->SV[i][j].value;
 }
 m->SV[i][numInputDimensions].index = -1; //Assign the final node value
 m->SV[i][numInputDimensions].value = 0;
 }
 }
 
 //The SV have now been loaded so flag that they should be deleted
 m->free_sv = 1;
 
 return m;
}

bool SVM::deepCopyProblem( const struct svm_problem &source, struct svm_problem &target, const unsigned int numInputDimensions ) const{

 //Cleanup the target memory
 if( target.y != NULL ){
 delete[] target.y;
 target.y = NULL;
 }
 if( target.x != NULL ){
 for(int i=0; i<target.l; i++){
 delete[] target.x[i];
 target.x[i] = NULL;
 }
 }

 //Deep copy the source to the target
 target.l = source.l;

 if( source.x != NULL ){
 target.x = new svm_node*[ target.l ];
 for(int i=0; i<target.l; i++){
 target.x[i] = new svm_node[ numInputDimensions+1 ];
 for(unsigned int j=0; j<numInputDimensions+1; j++){
 target.x[i][j] = source.x[i][j];
 }
 }
 }

 if( source.y != NULL ){
 target.y = new Float[ target.l ];
 for(int i=0; i<target.l; i++){
 target.y[i] = source.y[i];
 }
 }

 return true;
}

bool SVM::deepCopyParam( const svm_parameter &source_param, svm_parameter &target_param ) const{

 //Cleanup any dynamic memory in the target
 if( target_param.weight_label != NULL ){
 delete[] target_param.weight_label;
 target_param.weight_label = NULL;
 }
 if( target_param.weight != NULL ){
 delete[] target_param.weight;
 target_param.weight = NULL;
 }

 //Copy the non dynamic variables
 target_param.svm_type = source_param.svm_type;
 target_param.kernel_type = source_param.kernel_type;
 target_param.degree = source_param.degree;
 target_param.gamma = source_param.gamma;
 target_param.coef0 = source_param.coef0;
 target_param.cache_size = source_param.cache_size;
 target_param.eps = source_param.eps;
 target_param.C = source_param.C;
 target_param.nr_weight = source_param.nr_weight;
 target_param.nu = source_param.nu;
 target_param.p = source_param.p;
 target_param.shrinking = source_param.shrinking;
 target_param.probability = source_param.probability;

 //Copy any dynamic memory
 if( source_param.weight_label != NULL ){

 }
 if( source_param.weight != NULL ){

 }

 return true;
}
 
bool SVM::loadLegacyModelFromFile( std::fstream &file ){
 
 std::string word;
 
 UINT numSV = 0;
 UINT halfNumClasses = 0;
 numInputDimensions = 0;
 
 //Init the memory for the model
 model = new svm_model;
 model->nr_class = 0;
 model->l = 0;
 model->SV = NULL;
 model->sv_coef = NULL;
 model->rho = NULL;
 model->probA = NULL;
 model->probB = NULL;
 model->label = NULL;
 model->nSV = NULL;
 model->label = NULL;
 model->nSV = NULL;
 model->free_sv = 0; //This will be set to 1 if everything is loaded OK
 
 //Init the memory for the parameters
 model->param.svm_type = 0;
 model->param.kernel_type = 0;
 model->param.degree = 0;
 model->param.gamma = 0;
 model->param.coef0 = 0;
 model->param.cache_size = 0;
 model->param.eps = 0;
 model->param.C = 0;
 model->param.nr_weight = 0;
 model->param.weight_label = NULL;
 model->param.weight = NULL;
 model->param.nu = 0;
 model->param.p = 0;
 model->param.shrinking = 0;
 model->param.probability = 1;
 
 //Load the model type
 file >> word;
 if(word != "ModelType:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find ModelType header!" << std::endl;
 clear();
 return false;
 }
 file >> word;
 if( word == "C_SVC" ){
 model->param.svm_type = C_SVC;
 }else{
 if( word == "NU_SVC" ){
 model->param.svm_type = NU_SVC;
 }else{
 if( word == "ONE_CLASS" ){
 model->param.svm_type = ONE_CLASS;
 }else{
 if( word == "EPSILON_SVR" ){
 model->param.svm_type = EPSILON_SVR;
 }else{
 if( word == "NU_SVR" ){
 model->param.svm_type = NU_SVR;
 }else{
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find SVM type!" << std::endl;
 clear();
 return false;
 }
 }
 }
 }
 }
 
 //Load the model type
 file >> word;
 if(word != "KernelType:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find kernel type!" << std::endl;
 clear();
 return false;
 }
 file >> word;
 if( word == "LINEAR" ){
 model->param.kernel_type = LINEAR;
 }else{
 if( word == "POLYNOMIAL" ){
 model->param.kernel_type = POLY;
 }else{
 if( word == "RBF" ){
 model->param.kernel_type = RBF;
 }else{
 if( word == "SIGMOID" ){
 model->param.kernel_type = SIGMOID;
 }else{
 if( word == "PRECOMPUTED" ){
 model->param.kernel_type = PRECOMPUTED;
 }else{
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find kernel type!" << std::endl;
 clear();
 return false;
 }
 }
 }
 }
 }
 
 //Load the degree
 file >> word;
 if(word != "Degree:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find Degree header!" << std::endl;
 clear();
 return false;
 }
 file >> model->param.degree;
 
 //Load the gamma
 file >> word;
 if(word != "Gamma:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find Gamma header!" << std::endl;
 clear();
 return false;
 }
 file >> model->param.gamma;
 
 //Load the Coef0
 file >> word;
 if(word != "Coef0:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find Coef0 header!" << std::endl;
 clear();
 return false;
 }
 file >> model->param.coef0;
 
 //Load the NumberOfClasses
 file >> word;
 if(word != "NumberOfClasses:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find NumberOfClasses header!" << std::endl;
 clear();
 return false;
 }
 file >> numClasses;
 
 //Load the NumberOfSupportVectors
 file >> word;
 if(word != "NumberOfSupportVectors:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find NumberOfSupportVectors header!" << std::endl;
 clear();
 return false;
 }
 file >> numSV;
 
 //Load the NumberOfFeatures
 file >> word;
 if(word != "NumberOfFeatures:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find NumberOfFeatures header!" << std::endl;
 clear();
 return false;
 }
 file >> numInputDimensions;
 
 //Load the UseShrinking
 file >> word;
 if(word != "UseShrinking:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find UseShrinking header!" << std::endl;
 clear();
 return false;
 }
 file >> model->param.shrinking;
 
 //Load the UseProbability
 file >> word;
 if(word != "UseProbability:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find UseProbability header!" << std::endl;
 clear();
 return false;
 }
 file >> model->param.probability;
 
 //Load the UseScaling
 file >> word;
 if(word != "UseScaling:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find UseScaling header!" << std::endl;
 clear();
 return false;
 }
 file >> useScaling;
 
 //Load the Ranges
 file >> word;
 if(word != "Ranges:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find Ranges header!" << std::endl;
 clear();
 return false;
 }
 //Setup the memory for the ranges
 ranges.clear();
 ranges.resize(numInputDimensions);
 
 for(UINT i=0; i<ranges.size(); i++){
 file >> ranges[i].minValue;
 file >> ranges[i].maxValue;
 }
 
 //Setup the values
 halfNumClasses = numClasses*(numClasses-1)/2;
 model->nr_class = numClasses;
 model->l = numSV;
 
 //Load the RHO
 file >> word;
 if(word != "RHO:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find RHO header!" << std::endl;
 clear();
 return false;
 }
 model->rho = new Float[ halfNumClasses ];
 for(UINT i=0;i<numClasses*(numClasses-1)/2;i++) file >> model->rho[i];
 
 //See if we can load the Labels
 file >> word;
 if(word != "Label:"){
 model->label = NULL;
 }else{
 model->label = new int[ numClasses ];
 for(UINT i=0;i<numClasses;i++) file >> model->label[i];
 //We only need to read a new line if we found the label!
 file >> word;
 }
 
 //See if we can load the ProbA
 //We don't need to read another line here
 if(word != "ProbA:"){
 model->probA = NULL;
 }else{
 model->probA = new Float[ halfNumClasses ];
 for(UINT i=0;i<numClasses*(numClasses-1)/2;i++) file >> model->probA[i];
 //We only need to read a new line if we found the label!
 file >> word;
 }
 
 //See if we can load the ProbB
 //We don't need to read another line here
 if(word != "ProbB:"){
 model->probB = NULL;
 }else{
 model->probB = new Float[ halfNumClasses ];
 for(UINT i=0;i<numClasses*(numClasses-1)/2;i++) file >> model->probB[i];
 //We only need to read a new line if we found the label!
 file >> word;
 }
 
 //See if we can load the NumSupportVectorsPerClass
 //We don't need to read another line here
 if(word != "NumSupportVectorsPerClass:"){
 model->nSV = NULL;
 }else{
 model->nSV = new int[ numClasses ];
 for(UINT i=0;i<numClasses;i++) file >> model->nSV[i];
 //We only need to read a new line if we found the label!
 file >> word;
 }
 
 //Load the SupportVectors
 //We don't need to read another line here
 if(word != "SupportVectors:"){
 errorLog << "loadLegacyModelFromFile(fstream &file) - Failed to find SupportVectors header!" << std::endl;
 clear();
 return false;
 }
 
 //Setup the memory
 model->sv_coef = new Float*[numClasses-1];
 for(UINT j=0;j<numClasses-1;j++) model->sv_coef[j] = new Float[numSV];
 model->SV = new svm_node*[numSV];
 
 for(UINT i=0; i<numSV; i++){
 for(UINT j=0; j<numClasses-1; j++){
 file >> model->sv_coef[j][i];
 }
 
 model->SV[i] = new svm_node[numInputDimensions+1];
 
 if(model->param.kernel_type == PRECOMPUTED) file >> model->SV[i][0].value;
 else{
 for(UINT j=0; j<numInputDimensions; j++){
 file >> model->SV[i][j].index;
 file >> model->SV[i][j].value;
 }
 model->SV[i][numInputDimensions].index = -1; //Assign the final node value
 model->SV[i][numInputDimensions].value = 0;
 }
 }
 
 //Set the class labels
 this->numClasses = getNumClasses();
 classLabels.resize(getNumClasses());
 for(UINT k=0; k<getNumClasses(); k++){
 classLabels[k] = model->label[k];
 }
 
 //The SV have now been loaded so flag that they should be deleted
 model->free_sv = 1;
 
 //Finally, flag that the model has been trained to show it has been loaded and can be used for prediction
 trained = true;
 
 return true;
}
 
GRT_END_NAMESPACE