ClassificationData.cpp

/*
GRT MIT License
Copyright (c) <2012> <Nicholas Gillian, Media Lab, MIT>

Permission is hereby granted, free of charge, to any person obtaining a copy of this software
and associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial
portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

#include "ClassificationData.h"

GRT_BEGIN_NAMESPACE

ClassificationData::ClassificationData(const UINT numDimensions,const std::string datasetName,const std::string infoText){
 this->datasetName = datasetName;
 this->numDimensions = numDimensions;
 this->infoText = infoText;
 totalNumSamples = 0;
 crossValidationSetup = false;
 useExternalRanges = false;
 allowNullGestureClass = true;
 if( numDimensions > 0 ) setNumDimensions( numDimensions );
 infoLog.setProceedingText("[ClassificationData]");
 debugLog.setProceedingText("[DEBUG ClassificationData]");
 errorLog.setProceedingText("[ERROR ClassificationData]");
 warningLog.setProceedingText("[WARNING ClassificationData]");
}

ClassificationData::ClassificationData(const ClassificationData &rhs){
 *this = rhs;
}

ClassificationData::~ClassificationData(){
}
 
ClassificationData& ClassificationData::operator=(const ClassificationData &rhs){
 if( this != &rhs){
 this->datasetName = rhs.datasetName;
 this->infoText = rhs.infoText;
 this->numDimensions = rhs.numDimensions;
 this->totalNumSamples = rhs.totalNumSamples;
 this->kFoldValue = rhs.kFoldValue;
 this->crossValidationSetup = rhs.crossValidationSetup;
 this->useExternalRanges = rhs.useExternalRanges;
 this->allowNullGestureClass = rhs.allowNullGestureClass;
 this->externalRanges = rhs.externalRanges;
 this->classTracker = rhs.classTracker;
 this->data = rhs.data;
 this->crossValidationIndexs = rhs.crossValidationIndexs;
 this->infoLog = rhs.infoLog;
 this->debugLog = rhs.debugLog;
 this->errorLog = rhs.errorLog;
 this->warningLog = rhs.warningLog;
 }
 return *this;
}

void ClassificationData::clear(){
 totalNumSamples = 0;
 data.clear();
 classTracker.clear();
 crossValidationSetup = false;
 crossValidationIndexs.clear();
}

bool ClassificationData::setNumDimensions(const UINT numDimensions){

 if( numDimensions > 0 ){
 //Clear any previous training data
 clear();

 //Set the dimensionality of the data
 this->numDimensions = numDimensions;

 //Clear the external ranges
 useExternalRanges = false;
 externalRanges.clear();

 return true;
 }

 errorLog << "setNumDimensions(const UINT numDimensions) - The number of dimensions of the dataset must be greater than zero!" << std::endl;
 return false;
}

bool ClassificationData::setDatasetName(const std::string datasetName){

 //Make sure there are no spaces in the std::string
 if( datasetName.find(" ") == std::string::npos ){
 this->datasetName = datasetName;
 return true;
 }

 errorLog << "setDatasetName(const std::string datasetName) - The dataset name cannot contain any spaces!" << std::endl;
 return false;
}

bool ClassificationData::setInfoText(const std::string infoText){
 this->infoText = infoText;
 return true;
}

bool ClassificationData::setClassNameForCorrespondingClassLabel(const std::string className,const UINT classLabel){

 for(UINT i=0; i<classTracker.getSize(); i++){
 if( classTracker[i].classLabel == classLabel ){
 classTracker[i].className = className;
 return true;
 }
 }

 errorLog << "setClassNameForCorrespondingClassLabel(const std::string className,const UINT classLabel) - Failed to find class with label: " << classLabel << std::endl;
 return false;
}
 
bool ClassificationData::setAllowNullGestureClass(const bool allowNullGestureClass){
 this->allowNullGestureClass = allowNullGestureClass;
 return true;
}

bool ClassificationData::addSample(const UINT classLabel,const VectorFloat &sample){
 
 if( sample.getSize() != numDimensions ){
 errorLog << "addSample(const UINT classLabel, VectorFloat &sample) - the size of the new sample (" << sample.getSize() << ") does not match the number of dimensions of the dataset (" << numDimensions << ")" << std::endl;
 return false;
 }

 //The class label must be greater than zero (as zero is used for the null rejection class label
 if( classLabel == GRT_DEFAULT_NULL_CLASS_LABEL && !allowNullGestureClass ){
 errorLog << "addSample(const UINT classLabel, VectorFloat &sample) - the class label can not be 0!" << std::endl;
 return false;
 }

 //The dataset has changed so flag that any previous cross validation setup will now not work
 crossValidationSetup = false;
 crossValidationIndexs.clear();

 ClassificationSample newSample(classLabel,sample);
 data.push_back( newSample );
 totalNumSamples++;

 if( classTracker.getSize() == 0 ){
 ClassTracker tracker(classLabel,1);
 classTracker.push_back(tracker);
 }else{
 bool labelFound = false;
 for(UINT i=0; i<classTracker.getSize(); i++){
 if( classLabel == classTracker[i].classLabel ){
 classTracker[i].counter++;
 labelFound = true;
 break;
 }
 }
 if( !labelFound ){
 ClassTracker tracker(classLabel,1);
 classTracker.push_back(tracker);
 }
 }

 //Update the class labels
 sortClassLabels();

 return true;
}
 
bool ClassificationData::removeSample( const UINT index ){
 
 if( totalNumSamples == 0 ){
 warningLog << "removeSample( const UINT index ) - Failed to remove sample, the training dataset is empty!" << std::endl;
 return false;
 }
 
 if( index >= totalNumSamples ){
 warningLog << "removeSample( const UINT index ) - Failed to remove sample, the index is out of bounds! Number of training samples: " << totalNumSamples << " index: " << index << std::endl;
 return false;
 }
 
 //The dataset has changed so flag that any previous cross validation setup will now not work
 crossValidationSetup = false;
 crossValidationIndexs.clear();
 
 //Find the corresponding class ID for the last training example
 UINT classLabel = data[ index ].getClassLabel();
 
 //Remove the training example from the buffer
 data.erase( data.begin()+index );
 
 totalNumSamples = data.getSize();
 
 //Remove the value from the counter
 for(size_t i=0; i<classTracker.getSize(); i++){
 if( classTracker[i].classLabel == classLabel ){
 classTracker[i].counter--;
 break;
 }
 }
 
 return true;
}

bool ClassificationData::removeLastSample(){
 
 if( totalNumSamples == 0 ){
 warningLog << "removeLastSample() - Failed to remove sample, the training dataset is empty!" << std::endl;
 return false;
 }

 return removeSample( totalNumSamples-1 );
}

bool ClassificationData::reserve(const UINT N){
 
 data.reserve( N );
 
 if( data.capacity() >= N ) return true;
 
 return false;
}
 
UINT ClassificationData::eraseAllSamplesWithClassLabel(const UINT classLabel){
 return removeClass( classLabel );
}
 
bool ClassificationData::addClass(const UINT classLabel,const std::string className){
 
 //Check to make sure the class label does not exist
 for(size_t i=0; i<classTracker.getSize(); i++){
 if( classTracker[i].classLabel == classLabel ){
 warningLog << "addClass(const UINT classLabel,const std::string className) - Failed to add class, it already exists! Class label: " << classLabel << std::endl;
 return false;
 }
 }
 
 //Add the class label to the class tracker
 classTracker.push_back( ClassTracker(classLabel,0,className) );
 
 //Sort the class labels
 sortClassLabels();
 
 return true;
}
 
UINT ClassificationData::removeClass(const UINT classLabel){
 
 UINT numExamplesRemoved = 0;
 UINT numExamplesToRemove = 0;
 
 //The dataset has changed so flag that any previous cross validation setup will now not work
 crossValidationSetup = false;
 crossValidationIndexs.clear();
 
 //Find out how many training examples we need to remove
 for(UINT i=0; i<classTracker.getSize(); i++){
 if( classTracker[i].classLabel == classLabel ){
 numExamplesToRemove = classTracker[i].counter;
 classTracker.erase(classTracker.begin()+i);
 break;
 }
 }
 
 //Remove the samples with the matching class ID
 if( numExamplesToRemove > 0 ){
 UINT i=0;
 while( numExamplesRemoved < numExamplesToRemove ){
 if( data[i].getClassLabel() == classLabel ){
 data.erase(data.begin()+i);
 numExamplesRemoved++;
 }else if( ++i == data.getSize() ) break;
 }
 }
 
 totalNumSamples = data.getSize();
 
 return numExamplesRemoved;
}

bool ClassificationData::relabelAllSamplesWithClassLabel(const UINT oldClassLabel,const UINT newClassLabel){
 bool oldClassLabelFound = false;
 bool newClassLabelAllReadyExists = false;
 UINT indexOfOldClassLabel = 0;
 UINT indexOfNewClassLabel = 0;

 //Find out how many training examples we need to relabel
 for(UINT i=0; i<classTracker.getSize(); i++){
 if( classTracker[i].classLabel == oldClassLabel ){
 indexOfOldClassLabel = i;
 oldClassLabelFound = true;
 }
 if( classTracker[i].classLabel == newClassLabel ){
 indexOfNewClassLabel = i;
 newClassLabelAllReadyExists = true;
 }
 }

 //If the old class label was not found then we can't do anything
 if( !oldClassLabelFound ){
 return false;
 }

 //Relabel the old class labels
 for(UINT i=0; i<totalNumSamples; i++){
 if( data[i].getClassLabel() == oldClassLabel ){
 data[i].setClassLabel(newClassLabel);
 }
 }

 //Update the class tracler
 if( newClassLabelAllReadyExists ){
 //Add the old sample count to the new sample count
 classTracker[ indexOfNewClassLabel ].counter += classTracker[ indexOfOldClassLabel ].counter;
 }else{
 //Create a new class tracker
 classTracker.push_back( ClassTracker(newClassLabel,classTracker[ indexOfOldClassLabel ].counter,classTracker[ indexOfOldClassLabel ].className) );
 }

 //Erase the old class tracker
 classTracker.erase( classTracker.begin() + indexOfOldClassLabel );

 //Sort the class labels
 sortClassLabels();

 return true;
}

bool ClassificationData::setExternalRanges(const Vector< MinMax > &externalRanges, const bool useExternalRanges){

 if( externalRanges.size() != numDimensions ) return false;

 this->externalRanges = externalRanges;
 this->useExternalRanges = useExternalRanges;

 return true;
}

bool ClassificationData::enableExternalRangeScaling(const bool useExternalRanges){
 if( externalRanges.getSize() == numDimensions ){
 this->useExternalRanges = useExternalRanges;
 return true;
 }
 return false;
}

bool ClassificationData::scale(const Float minTarget,const Float maxTarget){
 Vector< MinMax > ranges = getRanges();
 return scale(ranges,minTarget,maxTarget);
}

bool ClassificationData::scale(const Vector<MinMax> &ranges,const Float minTarget,const Float maxTarget){
 if( ranges.getSize() != numDimensions ) return false;

 //Scale the training data
 for(UINT i=0; i<totalNumSamples; i++){
 for(UINT j=0; j<numDimensions; j++){
 data[i][j] = grt_scale(data[i][j],ranges[j].minValue,ranges[j].maxValue,minTarget,maxTarget);
 }
 }

 return true;
}
 
bool ClassificationData::save(const std::string &filename) const{
 
 //Check if the file should be saved as a csv file
 if( Util::stringEndsWith( filename, ".csv" ) ){
 return saveDatasetToCSVFile( filename );
 }
 
 //Otherwise save it as a custom GRT file
 return saveDatasetToFile( filename );
}

bool ClassificationData::load(const std::string &filename){
 
 //Check if the file should be loaded as a csv file
 if( Util::stringEndsWith( filename, ".csv" ) ){
 return loadDatasetFromCSVFile( filename );
 }
 
 //Otherwise save it as a custom GRT file
 return loadDatasetFromFile( filename );
}

bool ClassificationData::saveDatasetToFile(const std::string &filename) const{

 std::fstream file;
 file.open(filename.c_str(), std::ios::out);

 if( !file.is_open() ){
 return false;
 }

 file << "GRT_LABELLED_CLASSIFICATION_DATA_FILE_V1.0\n";
 file << "DatasetName: " << datasetName << std::endl;
 file << "InfoText: " << infoText << std::endl;
 file << "NumDimensions: " << numDimensions << std::endl;
 file << "TotalNumExamples: " << totalNumSamples << std::endl;
 file << "NumberOfClasses: " << classTracker.size() << std::endl;
 file << "ClassIDsAndCounters: " << std::endl;

 for(UINT i=0; i<classTracker.size(); i++){
 file << classTracker[i].classLabel << "\t" << classTracker[i].counter << "\t" << classTracker[i].className << std::endl;
 }

 file << "UseExternalRanges: " << useExternalRanges << std::endl;

 if( useExternalRanges ){
 for(UINT i=0; i<externalRanges.size(); i++){
 file << externalRanges[i].minValue << "\t" << externalRanges[i].maxValue << std::endl;
 }
 }

 file << "Data:\n";

 for(UINT i=0; i<totalNumSamples; i++){
 file << data[i].getClassLabel();
 for(UINT j=0; j<numDimensions; j++){
 file << "\t" << data[i][j];
 }
 file << std::endl;
 }

 file.close();
 return true;
}

bool ClassificationData::loadDatasetFromFile(const std::string &filename){

 std::fstream file;
 file.open(filename.c_str(), std::ios::in);
 UINT numClasses = 0;
 clear();

 if( !file.is_open() ){
 errorLog << "loadDatasetFromFile(const std::string &filename) - could not open file!" << std::endl;
 return false;
 }

 std::string word;

 //Check to make sure this is a file with the Training File Format
 file >> word;
 if(word != "GRT_LABELLED_CLASSIFICATION_DATA_FILE_V1.0"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - could not find file header!" << std::endl;
 file.close();
 return false;
 }

 //Get the name of the dataset
 file >> word;
 if(word != "DatasetName:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find DatasetName header!" << std::endl;
 errorLog << word << std::endl;
 file.close();
 return false;
 }
 file >> datasetName;

 file >> word;
 if(word != "InfoText:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find InfoText header!" << std::endl;
 file.close();
 return false;
 }

 //Load the info text
 file >> word;
 infoText = "";
 while( word != "NumDimensions:" ){
 infoText += word + " ";
 file >> word;
 }

 //Get the number of dimensions in the training data
 if( word != "NumDimensions:" ){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find NumDimensions header!" << std::endl;
 file.close();
 return false;
 }
 file >> numDimensions;

 //Get the total number of training examples in the training data
 file >> word;
 if( word != "TotalNumTrainingExamples:" && word != "TotalNumExamples:" ){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find TotalNumTrainingExamples header!" << std::endl;
 file.close();
 return false;
 }
 file >> totalNumSamples;

 //Get the total number of classes in the training data
 file >> word;
 if(word != "NumberOfClasses:"){
 errorLog << "loadDatasetFromFile(string filename) - failed to find NumberOfClasses header!" << std::endl;
 file.close();
 return false;
 }
 file >> numClasses;

 //Resize the class counter buffer and load the counters
 classTracker.resize(numClasses);

 //Get the total number of classes in the training data
 file >> word;
 if(word != "ClassIDsAndCounters:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find ClassIDsAndCounters header!" << std::endl;
 file.close();
 return false;
 }

 for(UINT i=0; i<classTracker.getSize(); i++){
 file >> classTracker[i].classLabel;
 file >> classTracker[i].counter;
 file >> classTracker[i].className;
 }

 //Check if the dataset should be scaled using external ranges
 file >> word;
 if(word != "UseExternalRanges:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find UseExternalRanges header!" << std::endl;
 file.close();
 return false;
 }
 file >> useExternalRanges;

 //If we are using external ranges then load them
 if( useExternalRanges ){
 externalRanges.resize(numDimensions);
 for(UINT i=0; i<externalRanges.getSize(); i++){
 file >> externalRanges[i].minValue;
 file >> externalRanges[i].maxValue;
 }
 }

 //Get the main training data
 file >> word;
 if( word != "LabelledTrainingData:" && word != "Data:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find LabelledTrainingData header!" << std::endl;
 file.close();
 return false;
 }

 ClassificationSample tempSample( numDimensions );
 data.resize( totalNumSamples, tempSample );

 for(UINT i=0; i<totalNumSamples; i++){
 UINT classLabel = 0;
 VectorFloat sample(numDimensions,0);
 file >> classLabel;
 for(UINT j=0; j<numDimensions; j++){
 file >> sample[j];
 }
 data[i].set(classLabel, sample);
 }

 file.close();
 
 //Sort the class labels
 sortClassLabels();
 
 return true;
}

bool ClassificationData::saveDatasetToCSVFile(const std::string &filename) const{

 std::fstream file;
 file.open(filename.c_str(), std::ios::out );

 if( !file.is_open() ){
 return false;
 }

 //Write the data to the CSV file
 for(UINT i=0; i<totalNumSamples; i++){
 file << data[i].getClassLabel();
 for(UINT j=0; j<numDimensions; j++){
 file << "," << data[i][j];
 }
 file << std::endl;
 }

 file.close();

 return true;
}

bool ClassificationData::loadDatasetFromCSVFile(const std::string &filename,const UINT classLabelColumnIndex){

 numDimensions = 0;
 datasetName = "NOT_SET";
 infoText = "";

 //Clear any previous data
 clear();

 //Parse the CSV file
 FileParser parser;

 Timer timer;

 timer.start();
 
 if( !parser.parseCSVFile(filename,true) ){
 errorLog << "loadDatasetFromCSVFile(const std::string &filename,const UINT classLabelColumnIndex) - Failed to parse CSV file!" << std::endl;
 return false;
 }
 
 if( !parser.getConsistentColumnSize() ){
 errorLog << "loadDatasetFromCSVFile(const std::string &filename,const UINT classLabelColumnIndexe) - The CSV file does not have a consistent number of columns!" << std::endl;
 return false;
 }
 
 if( parser.getColumnSize() <= 1 ){
 errorLog << "loadDatasetFromCSVFile(const std::string &filename,const UINT classLabelColumnIndex) - The CSV file does not have enough columns! It should contain at least two columns!" << std::endl;
 return false;
 }
 
 //Set the number of dimensions
 numDimensions = parser.getColumnSize()-1;

 timer.start();

 //Reserve the memory for the data
 data.resize( parser.getRowSize(), ClassificationSample(numDimensions) );

 timer.start();
 
 //Loop over the samples and add them to the data set
 UINT classLabel = 0;
 UINT j = 0;
 UINT n = 0;
 totalNumSamples = parser.getRowSize();
 for(UINT i=0; i<totalNumSamples; i++){
 //Get the class label
 classLabel = grt_from_str< UINT >( parser[i][classLabelColumnIndex] );
 
 //Set the class label
 data[i].setClassLabel( classLabel );
 
 //Get the sample data
 j=0;
 n=0;
 while( j != numDimensions ){
 if( n != classLabelColumnIndex ){
 data[i][j++] = grt_from_str< Float >( parser[i][n] );
 }
 n++;
 }
 
 //Update the class tracker
 if( classTracker.size() == 0 ){
 ClassTracker tracker(classLabel,1);
 classTracker.push_back(tracker);
 }else{
 bool labelFound = false;
 const size_t numClasses = classTracker.size();
 for(size_t i=0; i<numClasses; i++){
 if( classLabel == classTracker[i].classLabel ){
 classTracker[i].counter++;
 labelFound = true;
 break;
 }
 }
 if( !labelFound ){
 ClassTracker tracker(classLabel,1);
 classTracker.push_back(tracker);
 }
 }
 }

 //Sort the class labels
 sortClassLabels();
 
 return true;
}
 
bool ClassificationData::printStats() const{

 std::cout << getStatsAsString();

 return true;
}

bool ClassificationData::sortClassLabels(){
 
 sort(classTracker.begin(),classTracker.end(),ClassTracker::sortByClassLabelAscending);

 return true;
}

ClassificationData ClassificationData::partition(const UINT trainingSizePercentage,const bool useStratifiedSampling){

 //Partitions the dataset into a training dataset (which is kept by this instance of the ClassificationData) and
 //a testing/validation dataset (which is return as a new instance of the ClassificationData). The trainingSizePercentage
 //therefore sets the size of the data which remains in this instance and the remaining percentage of data is then added to
 //the testing/validation dataset

 //The dataset has changed so flag that any previous cross validation setup will now not work
 crossValidationSetup = false;
 crossValidationIndexs.clear();

 ClassificationData trainingSet(numDimensions);
 ClassificationData testSet(numDimensions);
 trainingSet.setAllowNullGestureClass( allowNullGestureClass );
 testSet.setAllowNullGestureClass( allowNullGestureClass );
 Vector< UINT > indexs( totalNumSamples );

 //Create the random partion indexs
 Random random;
 UINT randomIndex = 0;

 if( useStratifiedSampling ){
 //Break the data into seperate classes
 Vector< Vector< UINT > > classData( getNumClasses() );

 //Add the indexs to their respective classes
 for(UINT i=0; i<totalNumSamples; i++){
 classData[ getClassLabelIndexValue( data[i].getClassLabel() ) ].push_back( i );
 }

 //Randomize the order of the indexs in each of the class index buffers
 for(UINT k=0; k<getNumClasses(); k++){
 UINT numSamples = classData[k].getSize();
 for(UINT x=0; x<numSamples; x++){
 //Pick a random index
 randomIndex = random.getRandomNumberInt(0,numSamples);

 //Swap the indexs
 SWAP(classData[k][ x ], classData[k][ randomIndex ]);
 }
 }
 
 //Reserve the memory
 UINT numTrainingSamples = 0;
 UINT numTestSamples = 0;
 
 for(UINT k=0; k<getNumClasses(); k++){
 UINT numTrainingExamples = (UINT) floor( Float(classData[k].size()) / 100.0 * Float(trainingSizePercentage) );
 UINT numTestExamples = ((UINT)classData[k].size())-numTrainingExamples;
 numTrainingSamples += numTrainingExamples;
 numTestSamples += numTestExamples;
 }
 
 trainingSet.reserve( numTrainingSamples );
 testSet.reserve( numTestSamples );

 //Loop over each class and add the data to the trainingSet and testSet
 for(UINT k=0; k<getNumClasses(); k++){
 UINT numTrainingExamples = (UINT) floor( Float(classData[k].getSize()) / 100.0 * Float(trainingSizePercentage) );

 //Add the data to the training and test sets
 for(UINT i=0; i<numTrainingExamples; i++){
 trainingSet.addSample( data[ classData[k][i] ].getClassLabel(), data[ classData[k][i] ].getSample() );
 }
 for(UINT i=numTrainingExamples; i<classData[k].getSize(); i++){
 testSet.addSample( data[ classData[k][i] ].getClassLabel(), data[ classData[k][i] ].getSample() );
 }
 }
 }else{

 const UINT numTrainingExamples = (UINT) floor( Float(totalNumSamples) / 100.0 * Float(trainingSizePercentage) );
 //Create the random partion indexs
 Random random;
 UINT randomIndex = 0;
 for(UINT i=0; i<totalNumSamples; i++) indexs[i] = i;
 std::random_shuffle(indexs.begin(), indexs.end());
 
 //Reserve the memory
 trainingSet.reserve( numTrainingExamples );
 testSet.reserve( totalNumSamples-numTrainingExamples );

 //Add the data to the training and test sets
 for(UINT i=0; i<numTrainingExamples; i++){
 trainingSet.addSample( data[ indexs[i] ].getClassLabel(), data[ indexs[i] ].getSample() );
 }
 for(UINT i=numTrainingExamples; i<totalNumSamples; i++){
 testSet.addSample( data[ indexs[i] ].getClassLabel(), data[ indexs[i] ].getSample() );
 }
 }

 //Overwrite the training data in this instance with the training data of the trainingSet
 *this = trainingSet;

 //Sort the class labels in this dataset
 sortClassLabels();

 //Sort the class labels of the test dataset
 testSet.sortClassLabels();

 return testSet;
}

bool ClassificationData::merge(const ClassificationData &labelledData){

 if( labelledData.getNumDimensions() != numDimensions ){
 errorLog << "merge(const ClassificationData &labelledData) - The number of dimensions in the labelledData (" << labelledData.getNumDimensions() << ") does not match the number of dimensions of this dataset (" << numDimensions << ")" << std::endl;
 return false;
 }

 //The dataset has changed so flag that any previous cross validation setup will now not work
 crossValidationSetup = false;
 crossValidationIndexs.clear();
 
 //Reserve the memory
 reserve( getNumSamples() + labelledData.getNumSamples() );

 //Add the data from the labelledData to this instance
 for(UINT i=0; i<labelledData.getNumSamples(); i++){
 addSample(labelledData[i].getClassLabel(), labelledData[i].getSample());
 }

 //Set the class names from the dataset
 Vector< ClassTracker > classTracker = labelledData.getClassTracker();
 for(UINT i=0; i<classTracker.size(); i++){
 setClassNameForCorrespondingClassLabel(classTracker[i].className, classTracker[i].classLabel);
 }

 //Sort the class labels
 sortClassLabels();

 return true;
}

bool ClassificationData::spiltDataIntoKFolds(const UINT K,const bool useStratifiedSampling){

 crossValidationSetup = false;
 crossValidationIndexs.clear();

 //K can not be zero
 if( K > totalNumSamples ){
 errorLog << "spiltDataIntoKFolds(const UINT K,const bool useStratifiedSampling) - K can not be zero!" << std::endl;
 return false;
 }

 //K can not be larger than the number of examples
 if( K > totalNumSamples ){
 errorLog << "spiltDataIntoKFolds(const UINT K,const bool useStratifiedSampling) - K can not be larger than the total number of samples in the dataset!" << std::endl;
 return false;
 }

 //K can not be larger than the number of examples in a specific class if the stratified sampling option is true
 if( useStratifiedSampling ){
 for(UINT c=0; c<classTracker.size(); c++){
 if( K > classTracker[c].counter ){
 errorLog << "spiltDataIntoKFolds(const UINT K,const bool useStratifiedSampling) - K can not be larger than the number of samples in any given class!" << std::endl;
 return false;
 }
 }
 }

 //Setup the dataset for k-fold cross validation
 kFoldValue = K;
 Vector< UINT > indexs( totalNumSamples );

 //Work out how many samples are in each fold, the last fold might have more samples than the others
 UINT numSamplesPerFold = (UINT) floor( totalNumSamples/Float(K) );

 //Add the random indexs to each fold
 crossValidationIndexs.resize(K);

 //Create the random partion indexs
 Random random;
 UINT randomIndex = 0;

 if( useStratifiedSampling ){
 //Break the data into seperate classes
 Vector< Vector< UINT > > classData( getNumClasses() );

 //Add the indexs to their respective classes
 for(UINT i=0; i<totalNumSamples; i++){
 classData[ getClassLabelIndexValue( data[i].getClassLabel() ) ].push_back( i );
 }

 //Randomize the order of the indexs in each of the class index buffers
 for(UINT c=0; c<getNumClasses(); c++){
 UINT numSamples = (UINT)classData[c].size();
 for(UINT x=0; x<numSamples; x++){
 //Pick a random indexs
 randomIndex = random.getRandomNumberInt(0,numSamples);

 //Swap the indexs
 SWAP(classData[c][ x ] , classData[c][ randomIndex ]);
 }
 }

 //Loop over each of the k folds, at each fold add a sample from each class
 Vector< UINT >::iterator iter;
 for(UINT c=0; c<getNumClasses(); c++){
 iter = classData[ c ].begin();
 UINT k = 0;
 while( iter != classData[c].end() ){
 crossValidationIndexs[ k ].push_back( *iter );
 iter++;
 k++;
 k = k % K;
 }
 }

 }else{
 //Randomize the order of the data
 for(UINT i=0; i<totalNumSamples; i++) indexs[i] = i;
 for(UINT x=0; x<totalNumSamples; x++){
 //Pick a random index
 randomIndex = random.getRandomNumberInt(0,totalNumSamples);

 //Swap the indexs
 SWAP(indexs[ x ] , indexs[ randomIndex ]);
 }

 UINT counter = 0;
 UINT foldIndex = 0;
 for(UINT i=0; i<totalNumSamples; i++){
 //Add the index to the current fold
 crossValidationIndexs[ foldIndex ].push_back( indexs[i] );

 //Move to the next fold if ready
 if( ++counter == numSamplesPerFold && foldIndex < K-1 ){
 foldIndex++;
 counter = 0;
 }
 }
 }

 crossValidationSetup = true;
 return true;

}

ClassificationData ClassificationData::getTrainingFoldData(const UINT foldIndex) const{
 
 ClassificationData trainingData;
 trainingData.setNumDimensions( numDimensions );
 trainingData.setAllowNullGestureClass( allowNullGestureClass );

 if( !crossValidationSetup ){
 errorLog << "getTrainingFoldData(const UINT foldIndex) - Cross Validation has not been setup! You need to call the spiltDataIntoKFolds(UINT K,bool useStratifiedSampling) function first before calling this function!" << std::endl;
 return trainingData;
 }

 if( foldIndex >= kFoldValue ) return trainingData;

 //Add the class labels to make sure they all exist
 for(UINT k=0; k<getNumClasses(); k++){
 trainingData.addClass( classTracker[k].classLabel, classTracker[k].className );
 }

 //Add the data to the training set, this will consist of all the data that is NOT in the foldIndex
 UINT index = 0;
 for(UINT k=0; k<kFoldValue; k++){
 if( k != foldIndex ){
 for(UINT i=0; i<crossValidationIndexs[k].getSize(); i++){

 index = crossValidationIndexs[k][i];
 trainingData.addSample( data[ index ].getClassLabel(), data[ index ].getSample() );
 }
 }
 }

 //Sort the class labels
 trainingData.sortClassLabels();

 return trainingData;
}

ClassificationData ClassificationData::getTestFoldData(const UINT foldIndex) const{
 
 ClassificationData testData;
 testData.setNumDimensions( numDimensions );
 testData.setAllowNullGestureClass( allowNullGestureClass );

 if( !crossValidationSetup ) return testData;

 if( foldIndex >= kFoldValue ) return testData;

 //Add the class labels to make sure they all exist
 for(UINT k=0; k<getNumClasses(); k++){
 testData.addClass( classTracker[k].classLabel, classTracker[k].className );
 }
 
 testData.reserve( crossValidationIndexs[ foldIndex ].getSize() );

 //Add the data to the test fold
 UINT index = 0;
 for(UINT i=0; i<crossValidationIndexs[ foldIndex ].getSize(); i++){

 index = crossValidationIndexs[ foldIndex ][i];
 testData.addSample( data[ index ].getClassLabel(), data[ index ].getSample() );
 }
 
 //Sort the class labels
 testData.sortClassLabels();

 return testData;
}

ClassificationData ClassificationData::getClassData(const UINT classLabel) const{
 
 ClassificationData classData;
 classData.setNumDimensions( this->numDimensions );
 classData.setAllowNullGestureClass( allowNullGestureClass );
 
 //Reserve the memory for the class data
 for(UINT i=0; i<classTracker.getSize(); i++){
 if( classTracker[i].classLabel == classLabel ){
 classData.reserve( classTracker[i].counter );
 break;
 }
 }

 for(UINT i=0; i<totalNumSamples; i++){
 if( data[i].getClassLabel() == classLabel ){
 classData.addSample(classLabel, data[i].getSample());
 }
 }

 return classData;
}
 
ClassificationData ClassificationData::getBootstrappedDataset(UINT numSamples,bool balanceDataset) const{
 
 Random rand;
 ClassificationData newDataset;
 newDataset.setNumDimensions( getNumDimensions() );
 newDataset.setAllowNullGestureClass( allowNullGestureClass );
 newDataset.setExternalRanges( externalRanges, useExternalRanges );
 
 if( numSamples == 0 ) numSamples = totalNumSamples;
 
 newDataset.reserve( numSamples );

 const UINT K = getNumClasses(); 
 
 //Add all the class labels to the new dataset to ensure the dataset has a list of all the labels
 for(UINT k=0; k<K; k++){
 newDataset.addClass( classTracker[k].classLabel );
 }

 if( balanceDataset ){
 //Group the class indexs
 Vector< Vector< UINT > > classIndexs( K );
 for(UINT i=0; i<totalNumSamples; i++){
 classIndexs[ getClassLabelIndexValue( data[i].getClassLabel() ) ].push_back( i );
 }

 //Get the class with the minimum number of examples
 UINT numSamplesPerClass = (UINT)floor( numSamples / Float(K) );

 //Randomly select the training samples from each class
 UINT classIndex = 0;
 UINT classCounter = 0;
 UINT randomIndex = 0;
 for(UINT i=0; i<numSamples; i++){
 randomIndex = rand.getRandomNumberInt(0, (UINT)classIndexs[ classIndex ].size() );
 randomIndex = classIndexs[ classIndex ][ randomIndex ];
 newDataset.addSample(data[ randomIndex ].getClassLabel(), data[ randomIndex ].getSample());
 if( classCounter++ >= numSamplesPerClass && classIndex+1 < K ){
 classCounter = 0;
 classIndex++;
 }
 }

 }else{
 //Randomly select the training samples to add to the new data set
 UINT randomIndex;
 for(UINT i=0; i<numSamples; i++){
 randomIndex = rand.getRandomNumberInt(0, totalNumSamples);
 newDataset.addSample( data[randomIndex].getClassLabel(), data[randomIndex].getSample() );
 }
 }

 //Sort the class labels so they are in order
 newDataset.sortClassLabels();
 
 return newDataset;
}

RegressionData ClassificationData::reformatAsRegressionData() const{

 //Turns the classification into a regression data to enable regression algorithms like the MLP to be used as a classifier
 //This sets the number of targets in the regression data equal to the number of classes in the classification data
 //The output of each regression training sample will then be all 0's, except for the index matching the classLabel, which will be 1
 //For this to work, the labelled classification data cannot have any samples with a classLabel of 0!
 RegressionData regressionData;

 if( totalNumSamples == 0 ){
 return regressionData;
 }

 const UINT numInputDimensions = numDimensions;
 const UINT numTargetDimensions = getNumClasses();
 regressionData.setInputAndTargetDimensions(numInputDimensions, numTargetDimensions);

 for(UINT i=0; i<totalNumSamples; i++){
 VectorFloat targetVector(numTargetDimensions,0);

 //Set the class index in the target Vector to 1 and all other values in the target Vector to 0
 UINT classLabel = data[i].getClassLabel();

 if( classLabel > 0 ){
 targetVector[ classLabel-1 ] = 1;
 }else{
 regressionData.clear();
 return regressionData;
 }

 regressionData.addSample(data[i].getSample(),targetVector);
 }

 return regressionData;
}

UnlabelledData ClassificationData::reformatAsUnlabelledData() const{

 UnlabelledData unlabelledData;

 if( totalNumSamples == 0 ){
 return unlabelledData;
 }

 unlabelledData.setNumDimensions( numDimensions );

 for(UINT i=0; i<totalNumSamples; i++){
 unlabelledData.addSample( data[i].getSample() );
 }

 return unlabelledData;
}

UINT ClassificationData::getMinimumClassLabel() const{
 UINT minClassLabel = grt_numeric_limits< UINT >::max();

 for(UINT i=0; i<classTracker.getSize(); i++){
 if( classTracker[i].classLabel < minClassLabel ){
 minClassLabel = classTracker[i].classLabel;
 }
 }

 return minClassLabel;
}


UINT ClassificationData::getMaximumClassLabel() const{
 UINT maxClassLabel = 0;

 for(UINT i=0; i<classTracker.getSize(); i++){
 if( classTracker[i].classLabel > maxClassLabel ){
 maxClassLabel = classTracker[i].classLabel;
 }
 }

 return maxClassLabel;
}

UINT ClassificationData::getClassLabelIndexValue(UINT classLabel) const{
 for(UINT k=0; k<classTracker.getSize(); k++){
 if( classTracker[k].classLabel == classLabel ){
 return k;
 }
 }
 warningLog << "getClassLabelIndexValue(UINT classLabel) - Failed to find class label: " << classLabel << " in class tracker!" << std::endl;
 return 0;
}

std::string ClassificationData::getClassNameForCorrespondingClassLabel(UINT classLabel) const{

 for(UINT i=0; i<classTracker.getSize(); i++){
 if( classTracker[i].classLabel == classLabel ){
 return classTracker[i].className;
 }
 }

 return "CLASS_LABEL_NOT_FOUND";
}

std::string ClassificationData::getStatsAsString() const{
 std::string statsText;
 statsText += "DatasetName:\t" + datasetName + "\n";
 statsText += "DatasetInfo:\t" + infoText + "\n";
 statsText += "Number of Dimensions:\t" + Util::toString( numDimensions ) + "\n";
 statsText += "Number of Samples:\t" + Util::toString( totalNumSamples ) + "\n";
 statsText += "Number of Classes:\t" + Util::toString( getNumClasses() ) + "\n";
 statsText += "ClassStats:\n";

 for(UINT k=0; k<getNumClasses(); k++){
 statsText += "ClassLabel:\t" + Util::toString( classTracker[k].classLabel );
 statsText += "\tNumber of Samples:\t" + Util::toString(classTracker[k].counter);
 statsText += "\tClassName:\t" + classTracker[k].className + "\n";
 }

 Vector< MinMax > ranges = getRanges();

 statsText += "Dataset Ranges:\n";
 for(UINT j=0; j<ranges.size(); j++){
 statsText += "[" + Util::toString( j+1 ) + "] Min:\t" + Util::toString( ranges[j].minValue ) + "\tMax: " + Util::toString( ranges[j].maxValue ) + "\n";
 }

 return statsText;
}

Vector<MinMax> ClassificationData::getRanges() const{

 //If the dataset should be scaled using the external ranges then return the external ranges
 if( useExternalRanges ) return externalRanges;

 Vector< MinMax > ranges(numDimensions);

 //Otherwise return the min and max values for each column in the dataset
 if( totalNumSamples > 0 ){
 for(UINT j=0; j<numDimensions; j++){
 ranges[j].minValue = data[0][j];
 ranges[j].maxValue = data[0][j];
 for(UINT i=0; i<totalNumSamples; i++){
 if( data[i][j] < ranges[j].minValue ){ ranges[j].minValue = data[i][j]; } //Search for the min value
 else if( data[i][j] > ranges[j].maxValue ){ ranges[j].maxValue = data[i][j]; } //Search for the max value
 }
 }
 }
 return ranges;
}

Vector< UINT > ClassificationData::getClassLabels() const{
 Vector< UINT > classLabels( getNumClasses(), 0 );

 if( getNumClasses() == 0 ) return classLabels;

 for(UINT i=0; i<getNumClasses(); i++){
 classLabels[i] = classTracker[i].classLabel;
 }

 return classLabels;
}

Vector< UINT > ClassificationData::getNumSamplesPerClass() const{
 Vector< UINT > classSampleCounts( getNumClasses(), 0 );

 if( getNumSamples() == 0 ) return classSampleCounts;

 for(UINT i=0; i<getNumClasses(); i++){
 classSampleCounts[i] = classTracker[i].counter;
 }

 return classSampleCounts;
}

VectorFloat ClassificationData::getMean() const{
 
 VectorFloat mean(numDimensions,0);
 
 for(UINT j=0; j<numDimensions; j++){
 for(UINT i=0; i<totalNumSamples; i++){
 mean[j] += data[i][j];
 }
 mean[j] /= Float(totalNumSamples);
 }
 
 return mean;
}

VectorFloat ClassificationData::getStdDev() const{
 
 VectorFloat mean = getMean();
 VectorFloat stdDev(numDimensions,0);
 
 for(UINT j=0; j<numDimensions; j++){
 for(UINT i=0; i<totalNumSamples; i++){
 stdDev[j] += SQR(data[i][j]-mean[j]);
 }
 stdDev[j] = sqrt( stdDev[j] / Float(totalNumSamples-1) );
 }
 
 return stdDev;
}

MatrixFloat ClassificationData::getClassHistogramData(UINT classLabel,UINT numBins) const{

 const UINT M = getNumSamples();
 const UINT N = getNumDimensions();

 Vector< MinMax > ranges = getRanges();
 VectorFloat binRange(N);
 for(UINT i=0; i<ranges.size(); i++){
 binRange[i] = (ranges[i].maxValue-ranges[i].minValue)/Float(numBins);
 }

 MatrixFloat histData(N,numBins);
 histData.setAllValues(0);

 Float norm = 0;
 for(UINT i=0; i<M; i++){
 if( data[i].getClassLabel() == classLabel ){
 for(UINT j=0; j<N; j++){
 UINT binIndex = 0;
 bool binFound = false;
 for(UINT k=0; k<numBins-1; k++){
 if( data[i][j] >= ranges[i].minValue + (binRange[j]*k) && data[i][j] >= ranges[i].minValue + (binRange[j]*(k+1)) ){
 binIndex = k;
 binFound = true;
 break;
 }
 }
 if( !binFound ) binIndex = numBins-1;
 histData[j][binIndex]++;
 }
 norm++;
 }
 }

 if( norm == 0 ) return histData;

 //Is this the best way to normalize a multidimensional histogram???
 for(UINT i=0; i<histData.getNumRows(); i++){
 for(UINT j=0; j<histData.getNumCols(); j++){
 histData[i][j] /= norm;
 }
 }

 return histData;
}

MatrixFloat ClassificationData::getClassMean() const{
 
 MatrixFloat mean(getNumClasses(),numDimensions);
 VectorFloat counter(getNumClasses(),0);
 
 mean.setAllValues( 0 );
 
 for(UINT i=0; i<totalNumSamples; i++){
 UINT classIndex = getClassLabelIndexValue( data[i].getClassLabel() );
 for(UINT j=0; j<numDimensions; j++){
 mean[classIndex][j] += data[i][j];
 }
 counter[ classIndex ]++;
 }
 
 for(UINT k=0; k<getNumClasses(); k++){
 for(UINT j=0; j<numDimensions; j++){
 mean[k][j] = counter[k] > 0 ? mean[k][j]/counter[k] : 0;
 }
 }
 
 return mean;
}

MatrixFloat ClassificationData::getClassStdDev() const{

 MatrixFloat mean = getClassMean();
 MatrixFloat stdDev(getNumClasses(),numDimensions);
 VectorFloat counter(getNumClasses(),0);
 
 stdDev.setAllValues( 0 );
 
 for(UINT i=0; i<totalNumSamples; i++){
 UINT classIndex = getClassLabelIndexValue( data[i].getClassLabel() );
 for(UINT j=0; j<numDimensions; j++){
 stdDev[classIndex][j] += SQR(data[i][j]-mean[classIndex][j]);
 }
 counter[ classIndex ]++;
 }
 
 for(UINT k=0; k<getNumClasses(); k++){
 for(UINT j=0; j<numDimensions; j++){
 stdDev[k][j] = sqrt( stdDev[k][j] / Float(counter[k]-1) );
 }
 }
 
 return stdDev;
}

MatrixFloat ClassificationData::getCovarianceMatrix() const{
 
 VectorFloat mean = getMean();
 MatrixFloat covariance(numDimensions,numDimensions);
 
 for(UINT j=0; j<numDimensions; j++){
 for(UINT k=0; k<numDimensions; k++){
 for(UINT i=0; i<totalNumSamples; i++){
 covariance[j][k] += (data[i][j]-mean[j]) * (data[i][k]-mean[k]) ;
 }
 covariance[j][k] /= Float(totalNumSamples-1);
 }
 }
 
 return covariance;
}

Vector< MatrixFloat > ClassificationData::getHistogramData(UINT numBins) const{
 const UINT K = getNumClasses();
 Vector< MatrixFloat > histData(K);

 for(UINT k=0; k<K; k++){
 histData[k] = getClassHistogramData( classTracker[k].classLabel, numBins );
 }

 return histData;
}
 
VectorFloat ClassificationData::getClassProbabilities() const {
 return getClassProbabilities( getClassLabels() );
}
 
VectorFloat ClassificationData::getClassProbabilities( const Vector< UINT > &classLabels ) const {
 const UINT K = (UINT)classLabels.size();
 const UINT N = getNumClasses();
 Float sum = 0;
 VectorFloat x(K,0);
 for(UINT k=0; k<K; k++){
 for(UINT n=0; n<N; n++){
 if( classLabels[k] == classTracker[n].classLabel ){
 x[k] = classTracker[n].counter;
 sum += classTracker[n].counter;
 break;
 }
 }
 }
 
 //Normalize the class probabilities
 if( sum > 0 ){
 for(UINT k=0; k<K; k++){
 x[k] /= sum;
 }
 }
 
 return x;
}

Vector< UINT > ClassificationData::getClassDataIndexes(UINT classLabel) const{

 const UINT M = getNumSamples();
 const UINT K = getNumClasses();
 UINT N = 0;

 //Get the number of samples in the class
 for(UINT k=0; k<K; k++){
 if( classTracker[k].classLabel == classLabel){
 N = classTracker[k].counter;
 break;
 }
 }

 UINT index = 0;
 Vector< UINT > classIndexes(N);
 for(UINT i=0; i<M; i++){
 if( data[i].getClassLabel() == classLabel ){
 classIndexes[index++] = i;
 }
 }

 return classIndexes;
}

MatrixDouble ClassificationData::getDataAsMatrixDouble() const{

 const UINT M = getNumSamples();
 const UINT N = getNumDimensions();
 MatrixDouble d(M,N);

 for(UINT i=0; i<M; i++){
 for(UINT j=0; j<N; j++){
 d[i][j] = data[i][j];
 }
 }

 return d;
}

MatrixFloat ClassificationData::getDataAsMatrixFloat() const {
 const UINT M = getNumSamples();
 const UINT N = getNumDimensions();
 MatrixFloat d(M,N);

 for(UINT i=0; i<M; i++){
 for(UINT j=0; j<N; j++){
 d[i][j] = data[i][j];
 }
 }

 return d;
}

bool ClassificationData::generateGaussDataset( const std::string filename, const UINT numSamples, const UINT numClasses, const UINT numDimensions, const Float range, const Float sigma ){
 
 Random random;
 
 //Generate a simple model that will be used to generate the main dataset
 MatrixFloat model(numClasses,numDimensions);
 for(UINT k=0; k<numClasses; k++){
 for(UINT j=0; j<numDimensions; j++){
 model[k][j] = random.getRandomNumberUniform(-range,range);
 }
 }
 
 //Use the model above to generate the main dataset
 ClassificationData data;
 data.setNumDimensions( numDimensions );
 
 for(UINT i=0; i<numSamples; i++){
 
 //Randomly select which class this sample belongs to
 UINT k = random.getRandomNumberInt( 0, numClasses );
 
 //Generate a sample using the model (+ some Gaussian noise)
 VectorFloat sample( numDimensions );
 for(UINT j=0; j<numDimensions; j++){
 sample[j] = model[k][j] + random.getRandomNumberGauss(0,sigma);
 }
 
 //By default in the GRT, the class label should not be 0, so add 1
 UINT classLabel = k + 1;
 
 //Add the labeled sample to the dataset
 data.addSample( classLabel, sample );
 }
 
 //Save the dataset to a CSV file
 return data.save( filename );
}

GRT_END_NAMESPACE