UnlabelledData.cpp

/*
GRT MIT License
Copyright (c) <2012> <Nicholas Gillian, Media Lab, MIT>

Permission is hereby granted, free of charge, to any person obtaining a copy of this software
and associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial
portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

#define GRT_DLL_EXPORTS
#include "UnlabelledData.h"

GRT_BEGIN_NAMESPACE

UnlabelledData::UnlabelledData(const UINT numDimensions,const std::string datasetName,const std::string infoText):debugLog("[DEBUG ULCD]"),errorLog("[ERROR ULCD]"),warningLog("[WARNING ULCD]"){
 this->datasetName = datasetName;
 this->numDimensions = numDimensions;
 this->infoText = infoText;
 totalNumSamples = 0;
 crossValidationSetup = false;
 useExternalRanges = false;
 if( numDimensions > 0 ) setNumDimensions( numDimensions );
}

UnlabelledData::UnlabelledData(const UnlabelledData &rhs):debugLog("[DEBUG ULCD]"),errorLog("[ERROR ULCD]"),warningLog("[WARNING ULCD]"){
 *this = rhs;
}

UnlabelledData::~UnlabelledData(){}
 
UnlabelledData& UnlabelledData::operator=(const UnlabelledData &rhs){
 if( this != &rhs){
 this->datasetName = rhs.datasetName;
 this->infoText = rhs.infoText;
 this->numDimensions = rhs.numDimensions;
 this->totalNumSamples = rhs.totalNumSamples;
 this->kFoldValue = rhs.kFoldValue;
 this->crossValidationSetup = rhs.crossValidationSetup;
 this->useExternalRanges = rhs.useExternalRanges;
 this->externalRanges = rhs.externalRanges;
 this->data = rhs.data;
 this->crossValidationIndexs = rhs.crossValidationIndexs;
 this->debugLog = rhs.debugLog;
 this->errorLog = rhs.errorLog;
 this->warningLog = rhs.warningLog;
 }
 return *this;
}

void UnlabelledData::clear(){
 totalNumSamples = 0;
 data.clear();
 crossValidationSetup = false;
 crossValidationIndexs.clear();
}

bool UnlabelledData::setNumDimensions(const UINT numDimensions){

 if( numDimensions > 0 ){
 //Clear any previous training data
 clear();

 //Set the dimensionality of the data
 this->numDimensions = numDimensions;

 //Clear the external ranges
 useExternalRanges = false;
 externalRanges.clear();

 return true;
 }
 return false;
}

bool UnlabelledData::setDatasetName(const std::string datasetName){

 //Make sure there are no spaces in the std::string
 if( datasetName.find(" ") == std::string::npos ){
 this->datasetName = datasetName;
 return true;
 }

 return false;
}

bool UnlabelledData::setInfoText(const std::string infoText){
 this->infoText = infoText;
 return true;
}

bool UnlabelledData::addSample(const VectorFloat &sample){
 
 if( sample.size() != numDimensions ) return false;

 //The dataset has changed so flag that any previous cross validation setup will now not work
 crossValidationSetup = false;
 crossValidationIndexs.clear();

 data.push_back( sample );
 totalNumSamples++;

 return true;
}

bool UnlabelledData::removeLastSample(){

 if( totalNumSamples > 0 ){

 //The dataset has changed so flag that any previous cross validation setup will now not work
 crossValidationSetup = false;
 crossValidationIndexs.clear();

 //If there is only one sample then we just need to clear the buffer
 if( totalNumSamples == 1 ){
 data.clear();
 return true;
 }
 
 data.erase( data.begin()+data.size()-1 );

 return true;

 }else return false;

}

bool UnlabelledData::reserve(const UINT N){
 
 data.reserve( N );
 
 if( data.capacity() >= N ) return true;
 
 return false;
}
 
bool UnlabelledData::setExternalRanges(const Vector< MinMax > &externalRanges,const bool useExternalRanges){

 if( externalRanges.size() != numDimensions ) return false;

 this->externalRanges = externalRanges;
 this->useExternalRanges = useExternalRanges;

 return true;
}

bool UnlabelledData::enableExternalRangeScaling(const bool useExternalRanges){
 if( externalRanges.size() == numDimensions ){
 this->useExternalRanges = useExternalRanges;
 return true;
 }
 return false;
}

bool UnlabelledData::scale(const Float minTarget,const Float maxTarget){
 Vector< MinMax > ranges = getRanges();
 return scale(ranges,minTarget,maxTarget);
}

bool UnlabelledData::scale(const Vector<MinMax> &ranges,const Float minTarget,const Float maxTarget){
 if( ranges.size() != numDimensions ) return false;

 //Scale the training data
 for(UINT i=0; i<totalNumSamples; i++){
 for(UINT j=0; j<numDimensions; j++){
 data[i][j] = Util::scale(data[i][j],ranges[j].minValue,ranges[j].maxValue,minTarget,maxTarget);
 }
 }

 return true;
}
 
bool UnlabelledData::save(const std::string &filename) const{
 
 //Check if the file should be saved as a csv file
 if( Util::stringEndsWith( filename, ".csv" ) ){
 return saveDatasetToCSVFile( filename );
 }
 
 //Otherwise save it as a custom GRT file
 return saveDatasetToFile( filename );
}

bool UnlabelledData::load(const std::string &filename){
 
 //Check if the file should be loaded as a csv file
 if( Util::stringEndsWith( filename, ".csv" ) ){
 return loadDatasetFromCSVFile( filename );
 }
 
 //Otherwise save it as a custom GRT file
 return loadDatasetFromFile( filename );
}

bool UnlabelledData::saveDatasetToFile(const std::string &filename) const{

 std::fstream file;
 file.open(filename.c_str(), std::ios::out);

 if( !file.is_open() ){
 errorLog << "saveDatasetToFile(const std::string &filename) - Failed to open file!" << std::endl;
 return false;
 }

 file << "GRT_UNLABELLED_DATA_FILE_V1.0\n";
 file << "DatasetName: " << datasetName << std::endl;
 file << "InfoText: " << infoText << std::endl;
 file << "NumDimensions: " << numDimensions << std::endl;
 file << "TotalNumTrainingExamples: " << totalNumSamples << std::endl;

 file << "UseExternalRanges: " << useExternalRanges << std::endl;

 if( useExternalRanges ){
 for(UINT i=0; i<externalRanges.size(); i++){
 file << externalRanges[i].minValue << "\t" << externalRanges[i].maxValue << std::endl;
 }
 }

 file << "UnlabelledTrainingData:\n";

 for(UINT i=0; i<totalNumSamples; i++){
 for(UINT j=0; j<numDimensions; j++){
 if( j != 0 ) file << "\t";
 file << data[i][j];
 }
 file << std::endl;
 }

 file.close();
 return true;
}

bool UnlabelledData::loadDatasetFromFile(const std::string &filename){

 std::fstream file;
 file.open(filename.c_str(), std::ios::in);
 clear();

 if( !file.is_open() ){
 errorLog << "loadDatasetFromFile(const std::string &filename) - could not open file!" << std::endl;
 return false;
 }

 std::string word;

 //Check to make sure this is a file with the Training File Format
 file >> word;
 if( word != "GRT_UNLABELLED_DATA_FILE_V1.0" && word != "GRT_UNLABELLED_CLASSIFICATION_DATA_FILE_V1.0" ){
 errorLog << "loadDatasetFromFile(const std::string &filename) - could not find file header!" << std::endl;
 file.close();
 return false;
 }

 //Get the name of the dataset
 file >> word;
 if(word != "DatasetName:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find DatasetName!" << std::endl;
 file.close();
 return false;
 }
 file >> datasetName;

 file >> word;
 if(word != "InfoText:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find InfoText!" << std::endl;
 file.close();
 return false;
 }

 //Load the info text
 file >> word;
 infoText = "";
 while( word != "NumDimensions:" ){
 infoText += word + " ";
 file >> word;
 }

 //Get the number of dimensions in the training data
 if(word != "NumDimensions:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find DatasetName!" << std::endl;
 file.close();
 return false;
 }
 file >> numDimensions;

 //Get the total number of training examples in the training data
 file >> word;
 if(word != "TotalNumTrainingExamples:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find DatasetName!" << std::endl;
 file.close();
 return false;
 }
 file >> totalNumSamples;

 //Check if the dataset should be scaled using external ranges
 file >> word;
 if(word != "UseExternalRanges:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find DatasetName!" << std::endl;
 file.close();
 return false;
 }
 file >> useExternalRanges;

 //If we are using external ranges then load them
 if( useExternalRanges ){
 externalRanges.resize(numDimensions);
 for(UINT i=0; i<externalRanges.size(); i++){
 file >> externalRanges[i].minValue;
 file >> externalRanges[i].maxValue;
 }
 }

 //Get the main training data
 file >> word;
 if(word != "UnlabelledTrainingData:"){
 errorLog << "loadDatasetFromFile(const std::string &filename) - failed to find DatasetName!" << std::endl;
 file.close();
 return false;
 }
 data.resize( totalNumSamples, VectorFloat(numDimensions) );

 for(UINT i=0; i<totalNumSamples; i++){
 for(UINT j=0; j<numDimensions; j++){
 file >> data[i][j];
 }
 }

 file.close();
 return true;
}


bool UnlabelledData::saveDatasetToCSVFile(const std::string &filename) const{

 std::fstream file;
 file.open(filename.c_str(), std::ios::out );

 if( !file.is_open() ){
 errorLog << "saveDatasetToCSVFile(const std::string &filename) - Failed to open file!" << std::endl;
 return false;
 }

 //Write the data to the CSV file
 for(UINT i=0; i<totalNumSamples; i++){
 for(UINT j=0; j<numDimensions; j++){
 if( j != 0 ) file << ",";
 file << data[i][j];
 }
 file << std::endl;
 }

 file.close();

 return true;
}

bool UnlabelledData::loadDatasetFromCSVFile(const std::string &filename){

 std::string value;
 datasetName = "NOT_SET";
 infoText = "";

 //Clear any previous data
 clear();
 
 //Parse the CSV file
 FileParser parser;
 
 if( !parser.parseCSVFile(filename,true) ){
 errorLog << "loadDatasetFromCSVFile(const std::string &filename) - Failed to parse CSV file!" << std::endl;
 return false;
 }
 
 if( !parser.getConsistentColumnSize() ){
 errorLog << "loadDatasetFromCSVFile(const std::string &filename) - The CSV file does not have a consistent number of columns!" << std::endl;
 return false;
 }
 
 const UINT rows = parser.getRowSize();
 const UINT cols = parser.getColumnSize();
 
 //Setup the labelled classification data
 numDimensions = cols;
 
 //Reserve the data so we do not have to continually resize the memory
 data.reserve( rows );

 VectorFloat sample(numDimensions);
 for(UINT i=0; i<rows; i++){
 
 //Get the input vector
 for(UINT j=0; j<numDimensions; j++){
 sample[j] = Util::stringToFloat( parser[i][j] );
 }
 
 //Add the labelled sample to the dataset
 if( !addSample(sample) ){
 warningLog << "loadDatasetFromCSVFile(const std::string &filename) - Could not add sample " << i << " to the dataset!" << std::endl;
 }
 }
 
 return true;
}

UnlabelledData UnlabelledData::partition(const UINT trainingSizePercentage){
 return split( trainingSizePercentage );
}

UnlabelledData UnlabelledData::split(const UINT trainingSizePercentage){

 //Partitions the dataset into a training dataset (which is kept by this instance of the UnlabelledData) and
 //a testing/validation dataset (which is return as a new instance of the UnlabelledData). The trainingSizePercentage
 //therefore sets the size of the data which remains in this instance and the remaining percentage of data is then added to
 //the testing/validation dataset

 //The dataset has changed so flag that any previous cross validation setup will now not work
 crossValidationSetup = false;
 crossValidationIndexs.clear();

 const UINT numTrainingExamples = (UINT) floor( Float(totalNumSamples) / 100.0 * Float(trainingSizePercentage) );

 UnlabelledData trainingSet(numDimensions);
 UnlabelledData testSet(numDimensions);
 Vector< UINT > indexs( totalNumSamples );

 //Create the random partion indexs
 Random random;
 UINT randomIndex = 0;
 for(UINT i=0; i<totalNumSamples; i++) indexs[i] = i;
 for(UINT x=0; x<totalNumSamples; x++){
 //Pick a random index
 randomIndex = random.getRandomNumberInt(0,totalNumSamples);

 //Swap the indexs
 SWAP( indexs[ x ] , indexs[ randomIndex ] );
 }
 
 trainingSet.reserve( numTrainingExamples );
 testSet.reserve( totalNumSamples-numTrainingExamples );

 //Add the data to the training and test sets
 for(UINT i=0; i<numTrainingExamples; i++){
 trainingSet.addSample( data[ indexs[i] ] );
 }
 for(UINT i=numTrainingExamples; i<totalNumSamples; i++){
 testSet.addSample( data[ indexs[i] ] );
 }

 //Overwrite the training data in this instance with the training data of the trainingSet
 *this = trainingSet;

 return testSet;
}

bool UnlabelledData::merge(const UnlabelledData &unlabelledData){

 if( unlabelledData.getNumDimensions() != numDimensions ){
 errorLog << "merge(const UnlabelledData &unlabelledData) - The number of dimensions in the unlabelledData (" << unlabelledData.getNumDimensions() << ") does not match the number of dimensions of this dataset (" << numDimensions << ")" << std::endl;
 return false;
 }

 //The dataset has changed so flag that any previous cross validation setup will now not work
 crossValidationSetup = false;
 crossValidationIndexs.clear();
 
 reserve( getNumSamples() + unlabelledData.getNumSamples() );

 //Add the data from the labelledData to this instance
 for(UINT i=0; i<unlabelledData.getNumSamples(); i++){
 addSample( unlabelledData[i] );
 }

 return true;
}

bool UnlabelledData::spiltDataIntoKFolds(const UINT K){

 crossValidationSetup = false;
 crossValidationIndexs.clear();

 //K can not be zero
 if( K > totalNumSamples ){
 errorLog << "spiltDataIntoKFolds(const UINT K) - K can not be zero!" << std::endl;
 return false;
 }

 //K can not be larger than the number of examples
 if( K > totalNumSamples ){
 errorLog << "spiltDataIntoKFolds(const UINT K) - K can not be larger than the total number of samples in the dataset!" << std::endl;
 return false;
 }

 //Setup the dataset for k-fold cross validation
 kFoldValue = K;
 Vector< UINT > indexs( totalNumSamples );

 //Work out how many samples are in each fold, the last fold might have more samples than the others
 UINT numSamplesPerFold = (UINT) floor( totalNumSamples/Float(K) );

 //Add the random indexs to each fold
 crossValidationIndexs.resize(K);

 //Create the random partion indexs
 Random random;
 UINT randomIndex = 0;

 //Randomize the order of the data
 for(UINT i=0; i<totalNumSamples; i++) indexs[i] = i;
 for(UINT x=0; x<totalNumSamples; x++){
 //Pick a random index
 randomIndex = random.getRandomNumberInt(0,totalNumSamples);

 //Swap the indexs
 grt_swap( indexs[ x ] , indexs[ randomIndex ] );
 }

 UINT counter = 0;
 UINT foldIndex = 0;
 for(UINT i=0; i<totalNumSamples; i++){
 //Add the index to the current fold
 crossValidationIndexs[ foldIndex ].push_back( indexs[i] );

 //Move to the next fold if ready
 if( ++counter == numSamplesPerFold && foldIndex < K-1 ){
 foldIndex++;
 counter = 0;
 }
 }

 crossValidationSetup = true;
 return true;

}

UnlabelledData UnlabelledData::getTrainingFoldData(const UINT foldIndex) const{
 UnlabelledData trainingData;

 if( !crossValidationSetup ){
 errorLog << "getTrainingFoldData(const UINT foldIndex) - Cross Validation has not been setup! You need to call the spiltDataIntoKFolds(UINT K) function first before calling this function!" << std::endl;
 return trainingData;
 }

 if( foldIndex >= kFoldValue ) return trainingData;

 trainingData.setNumDimensions( numDimensions );
 
 //Work out how many samples will be in this fold
 UINT numSamples = 0;
 for(UINT k=0; k<kFoldValue; k++){
 if( k != foldIndex ){
 numSamples += (UINT)crossValidationIndexs[k].size();
 }
 }
 trainingData.reserve( numSamples );

 //Add the data to the training set, this will consist of all the data that is NOT in the foldIndex
 UINT index = 0;
 for(UINT k=0; k<kFoldValue; k++){
 if( k != foldIndex ){
 for(UINT i=0; i<crossValidationIndexs[k].size(); i++){

 index = crossValidationIndexs[k][i];
 trainingData.addSample( data[ index ] );
 }
 }
 }

 return trainingData;
}

UnlabelledData UnlabelledData::getTestFoldData(const UINT foldIndex) const{
 UnlabelledData testData;

 if( !crossValidationSetup ) return testData;

 if( foldIndex >= kFoldValue ) return testData;

 //Add the data to the training
 testData.setNumDimensions( numDimensions );
 
 //Work out how many samples will be in this fold
 UINT numSamples = (UINT)crossValidationIndexs[ foldIndex ].size();

 testData.reserve( numSamples );

 UINT index = 0;
 for(UINT i=0; i<crossValidationIndexs[ foldIndex ].size(); i++){

 index = crossValidationIndexs[ foldIndex ][i];
 testData.addSample( data[ index ] );
 }

 return testData;
}

std::string UnlabelledData::getStatsAsString() const{
 std::string statsText;
 statsText += "DatasetName:\t" + datasetName + "\n";
 statsText += "DatasetInfo:\t" + infoText + "\n";
 statsText += "Number of Dimensions:\t" + Util::toString( numDimensions ) + "\n";
 statsText += "Number of Samples:\t" + Util::toString( totalNumSamples ) + "\n";

 Vector< MinMax > ranges = getRanges();

 statsText += "Dataset Ranges:\n";
 for(UINT j=0; j<ranges.size(); j++){
 statsText += "[" + Util::toString( j+1 ) + "] Min:\t" + Util::toString( ranges[j].minValue ) + "\tMax: " + Util::toString( ranges[j].maxValue ) + "\n";
 }

 return statsText;
}

Vector<MinMax> UnlabelledData::getRanges() const{

 //If the dataset should be scaled using the external ranges then return the external ranges
 if( useExternalRanges ) return externalRanges;
 
 Vector< MinMax > ranges(numDimensions);

 //Otherwise return the min and max values for each column in the dataset
 if( totalNumSamples > 0 ){
 for(UINT j=0; j<numDimensions; j++){
 ranges[j].minValue = data[0][0];
 ranges[j].maxValue = data[0][0];
 for(UINT i=0; i<totalNumSamples; i++){
 if( data[i][j] < ranges[j].minValue ){ ranges[j].minValue = data[i][j]; } //Search for the min value
 else if( data[i][j] > ranges[j].maxValue ){ ranges[j].maxValue = data[i][j]; } //Search for the max value
 }
 }
 }
 return ranges;
}
 
Vector< VectorFloat > UnlabelledData::getData() const{
 return data;
}

MatrixDouble UnlabelledData::getDataAsMatrixDouble() const{
 const UINT rows = getNumSamples();
 const UINT cols = getNumDimensions();
 MatrixDouble d(rows,cols);
 
 for(UINT i=0; i<rows; i++){
 for(UINT j=0; j<cols; j++){
 d[i][j] = data[i][j];
 }
 }
 
 return d;
}

MatrixFloat UnlabelledData::getDataAsMatrixFloat() const {
 const UINT rows = getNumSamples();
 const UINT cols = getNumDimensions();
 MatrixFloat d(rows,cols);
 
 for(UINT i=0; i<rows; i++){
 for(UINT j=0; j<cols; j++){
 d[i][j] = data[i][j];
 }
 }
 
 return d;
}

GRT_END_NAMESPACE