2 #define GRT_DLL_EXPORTS
7 BernoulliRBM::BernoulliRBM(
const UINT numHiddenUnits,
const UINT maxNumEpochs,
const Float learningRate,
const Float learningRateUpdate,
const Float momentum,
const bool useScaling,
const bool randomiseTrainingOrder){
9 this->numHiddenUnits = numHiddenUnits;
10 this->maxNumEpochs = maxNumEpochs;
11 this->learningRate = learningRate;
12 this->learningRateUpdate = learningRateUpdate;
13 this->momentum = momentum;
14 this->useScaling = useScaling;
15 this->randomiseTrainingOrder = randomiseTrainingOrder;
16 randomizeWeightsForTraining =
true;
22 classType =
"BernoulliRBM";
23 debugLog.setProceedingText(
"[DEBUG BernoulliRBM]");
24 errorLog.setProceedingText(
"[ERROR BernoulliRBM]");
25 trainingLog.setProceedingText(
"[TRAINING BernoulliRBM]");
26 warningLog.setProceedingText(
"[WARNING BernoulliRBM]");
29 BernoulliRBM::~BernoulliRBM(){
35 if( !
predict_(inputData,outputData) ){
45 errorLog <<
"predict_(VectorFloat &inputData,VectorFloat &outputData) - Failed to run prediction - the model has not been trained." << std::endl;
49 if( inputData.size() != numVisibleUnits ){
50 errorLog <<
"predict_(VectorFloat &inputData,VectorFloat &outputData) - Failed to run prediction - the input data size (" << inputData.size() <<
")";
51 errorLog <<
" does not match the number of visible units (" << numVisibleUnits <<
"). " << std::endl;
55 if( outputData.size() != numHiddenUnits ){
56 outputData.
resize( numHiddenUnits );
61 for(UINT i=0; i<numVisibleUnits; i++){
62 inputData[i] = grt_scale(inputData[i],ranges[i].minValue,ranges[i].maxValue,0.0,1.0);
68 for(UINT i=0; i<numHiddenUnits; i++){
69 for(UINT j=0; j<numVisibleUnits; j++) {
70 x += weightsMatrix[i][j] * inputData[j];
72 outputData[i] = grt_sigmoid( x + hiddenLayerBias[i] );
81 errorLog <<
"predict_(const MatrixFloat &inputData,MatrixFloat &outputData,const UINT rowIndex) - Failed to run prediction - the model has not been trained." << std::endl;
85 if( inputData.
getNumCols() != numVisibleUnits ){
86 errorLog <<
"predict_(const MatrixFloat &inputData,MatrixFloat &outputData,const UINT rowIndex) -";
87 errorLog <<
" Failed to run prediction - the number of columns in the input matrix (" << inputData.
getNumCols() <<
")";
88 errorLog <<
" does not match the number of visible units (" << numVisibleUnits <<
")." << std::endl;
92 if( outputData.
getNumCols() != numHiddenUnits ){
93 errorLog <<
"predict_(const MatrixFloat &inputData,MatrixFloat &outputData,const UINT rowIndex) -";
94 errorLog <<
" Failed to run prediction - the number of columns in the output matrix (" << outputData.
getNumCols() <<
")";
95 errorLog <<
" does not match the number of hidden units (" << numHiddenUnits <<
")." << std::endl;
101 for(UINT j=0; j<numHiddenUnits; j++){
103 for(UINT i=0; i<numVisibleUnits; i++) {
104 x += weightsMatrix[j][i] * inputData[rowIndex][i];
106 outputData[rowIndex][j] = grt_sigmoid( x + hiddenLayerBias[j] );
114 const UINT numTrainingSamples = data.
getNumRows();
116 numOutputDimensions = numHiddenUnits;
117 numVisibleUnits = numInputDimensions;
119 trainingLog <<
"NumInputDimensions: " << numInputDimensions << std::endl;
120 trainingLog <<
"NumOutputDimensions: " << numOutputDimensions << std::endl;
122 if( randomizeWeightsForTraining ){
125 weightsMatrix.
resize(numHiddenUnits, numVisibleUnits);
127 Float a = 1.0 / numVisibleUnits;
128 for(UINT i=0; i<numHiddenUnits; i++) {
129 for(UINT j=0; j<numVisibleUnits; j++) {
135 visibleLayerBias.
resize( numVisibleUnits );
136 hiddenLayerBias.
resize( numHiddenUnits );
137 std::fill(visibleLayerBias.begin(),visibleLayerBias.end(),0);
138 std::fill(hiddenLayerBias.begin(),hiddenLayerBias.end(),0);
141 if( weightsMatrix.
getNumRows() != numHiddenUnits ){
142 errorLog <<
"train_(MatrixFloat &data) - Weights matrix row size does not match the number of hidden units!" << std::endl;
145 if( weightsMatrix.
getNumCols() != numVisibleUnits ){
146 errorLog <<
"train_(MatrixFloat &data) - Weights matrix row size does not match the number of visible units!" << std::endl;
149 if( visibleLayerBias.size() != numVisibleUnits ){
150 errorLog <<
"train_(MatrixFloat &data) - Visible layer bias size does not match the number of visible units!" << std::endl;
153 if( hiddenLayerBias.size() != numHiddenUnits ){
154 errorLog <<
"train_(MatrixFloat &data) - Hidden layer bias size does not match the number of hidden units!" << std::endl;
165 for(UINT i=0; i<numTrainingSamples; i++){
166 for(UINT j=0; j<numInputDimensions; j++){
167 data[i][j] = grt_scale(data[i][j], ranges[j].minValue, ranges[j].maxValue, 0.0, 1.0);
173 const UINT numBatches =
static_cast<UINT
>( ceil( Float(numTrainingSamples)/batchSize ) );
178 for(UINT i=0; i<numBatches; i++){
179 batchIndexs[i].startIndex = startIndex;
180 batchIndexs[i].endIndex = startIndex + batchSize;
183 if( batchIndexs[i].endIndex >= numTrainingSamples ){
184 batchIndexs[i].endIndex = numTrainingSamples;
188 batchIndexs[i].batchSize = batchIndexs[i].endIndex - batchIndexs[i].startIndex;
191 startIndex = batchIndexs[i].endIndex;
195 UINT i,j,n,epoch,noChangeCounter = 0;
197 Float alpha = learningRate;
203 TrainingResult trainingResult;
206 MatrixFloat tmpW( numHiddenUnits, numVisibleUnits );
215 MatrixFloat cDiff( numHiddenUnits, numVisibleUnits );
218 VectorFloat visibleLayerBiasVelocity( numVisibleUnits );
219 VectorFloat hiddenLayerBiasVelocity( numHiddenUnits );
223 std::fill(visibleLayerBiasVelocity.begin(),visibleLayerBiasVelocity.end(),0);
224 std::fill(hiddenLayerBiasVelocity.begin(),hiddenLayerBiasVelocity.end(),0);
227 for(UINT i=0; i<numTrainingSamples; i++) indexList[i] = i;
228 if( randomiseTrainingOrder ){
229 std::random_shuffle(indexList.begin(), indexList.end());
234 for(epoch=0; epoch<maxNumEpochs; epoch++) {
239 std::random_shuffle(batchIndexs.begin(),batchIndexs.end());
242 for(UINT k=0; k<numBatches; k+=batchStepSize){
245 v1.
resize( batchIndexs[k].batchSize, numVisibleUnits );
246 h1.
resize( batchIndexs[k].batchSize, numHiddenUnits );
247 v2.
resize( batchIndexs[k].batchSize, numVisibleUnits );
248 h2.
resize( batchIndexs[k].batchSize, numHiddenUnits );
259 Float *vlb_p = &visibleLayerBias[0];
260 Float *hlb_p = &hiddenLayerBias[0];
264 for(i=batchIndexs[k].startIndex; i<batchIndexs[k].endIndex; i++){
265 for(j=0; j<numVisibleUnits; j++){
266 v1_p[index][j] = data_p[ indexList[i] ][j];
272 for(i=0; i<numHiddenUnits; i++)
273 for(j=0; j<numVisibleUnits; j++)
274 wT_p[j][i] = w_p[i][j];
278 for(n=0; n<batchIndexs[k].batchSize; n++){
279 for(i=0; i<numHiddenUnits; i++){
280 h1_p[n][i] = sigmoidRandom( h1_p[n][i] + hlb_p[i] );
286 for(n=0; n<batchIndexs[k].batchSize; n++){
287 for(i=0; i<numVisibleUnits; i++){
288 v2_p[n][i] = sigmoidRandom( v2_p[n][i] + vlb_p[i] );
294 for(n=0; n<batchIndexs[k].batchSize; n++){
295 for(i=0; i<numHiddenUnits; i++){
296 h2_p[n][i] = grt_sigmoid( h2_p[n][i] + hlb_p[i] );
306 for(j=0; j<numVisibleUnits; j++){
308 for(i=0; i<batchIndexs[k].batchSize; i++){
309 vDiffSum[j] += vDiff[i][j];
315 for(j=0; j<numHiddenUnits; j++){
317 for(i=0; i<batchIndexs[k].batchSize; i++){
318 hDiffSum[j] += hDiff[i][j];
326 for(i=0; i<numHiddenUnits; i++){
327 for(j=0; j<numVisibleUnits; j++){
328 vW_p[i][j] = ((momentum * vW_p[i][j]) + (alpha * cDiff[i][j])) / batchIndexs[k].batchSize;
331 for(i=0; i<numVisibleUnits; i++){
332 visibleLayerBiasVelocity[i] = ((momentum * visibleLayerBiasVelocity[i]) + (alpha * vDiffSum[i])) / batchIndexs[k].batchSize;
334 for(i=0; i<numHiddenUnits; i++){
335 hiddenLayerBiasVelocity[i] = ((momentum * hiddenLayerBiasVelocity[i]) + (alpha * hDiffSum[i])) / batchIndexs[k].batchSize;
339 weightsMatrix.
add( vW );
342 for(i=0; i<numVisibleUnits; i++){
343 visibleLayerBias[i] += visibleLayerBiasVelocity[i];
347 for(i=0; i<numHiddenUnits; i++){
348 hiddenLayerBias[i] += hiddenLayerBiasVelocity[i];
353 for(i=0; i<batchIndexs[k].batchSize; i++){
354 for(j=0; j<numVisibleUnits; j++){
355 err += SQR( v1[i][j] - v2[i][j] );
359 error += err / batchIndexs[k].batchSize;
362 delta = lastError - error;
365 trainingLog <<
"Epoch: " << epoch+1 <<
"/" << maxNumEpochs;
366 trainingLog <<
" Epoch time: " << (timer.
getMilliSeconds()-startTime)/1000.0 <<
" seconds";
367 trainingLog <<
" Learning rate: " << alpha;
368 trainingLog <<
" Momentum: " << momentum;
369 trainingLog <<
" Average reconstruction error: " << error;
370 trainingLog <<
" Delta: " << delta << std::endl;
373 alpha *= learningRateUpdate;
375 trainingResult.setClassificationResult(epoch, error,
this);
376 trainingResults.push_back(trainingResult);
377 trainingResultsObserverManager.notifyObservers( trainingResult );
380 if( fabs(delta) < minChange ){
381 if( ++noChangeCounter >= minNumEpochs ){
382 trainingLog <<
"Stopping training. MinChange limit reached!" << std::endl;
385 }
else noChangeCounter = 0;
388 trainingLog <<
"Training complete after " << epoch <<
" epochs. Total training time: " << timer.
getMilliSeconds()/1000.0 <<
" seconds" << std::endl;
408 weightsMatrix.
clear();
409 weightsMatrix.
clear();
410 visibleLayerBias.clear();
411 hiddenLayerBias.clear();
421 randomizeWeightsForTraining =
true;
430 errorLog <<
"save(fstream &file) - The file is not open!" << std::endl;
435 file<<
"GRT_BERNOULLI_RBM_MODEL_FILE_V1.1\n";
438 errorLog <<
"save(fstream &file) - Failed to save base settings to file!" << std::endl;
442 file <<
"NumVisibleUnits: " << numVisibleUnits << std::endl;
443 file <<
"NumHiddenUnits: " << numHiddenUnits << std::endl;
444 file <<
"BatchSize: " << batchSize << std::endl;
445 file <<
"BatchStepSize: " << batchStepSize << std::endl;
446 file <<
"LearningRate: " << learningRate << std::endl;
447 file <<
"LearningRateUpdate: " << learningRateUpdate << std::endl;
448 file <<
"Momentum: " << momentum << std::endl;
449 file <<
"RandomizeWeightsForTraining: " << randomizeWeightsForTraining << std::endl;
451 file <<
"Ranges: \n";
452 for(UINT n=0; n<ranges.size(); n++){
453 file << ranges[n].minValue <<
"\t" << ranges[n].maxValue << std::endl;
458 file <<
"WeightsMatrix: " << std::endl;
459 for(UINT i=0; i<weightsMatrix.
getNumRows(); i++){
460 for(UINT j=0; j<weightsMatrix.
getNumCols(); j++){
461 file << weightsMatrix[i][j];
462 if( j < weightsMatrix.getNumCols()-1 ) file <<
" ";
467 file <<
"VisibleLayerBias: ";
468 for(
unsigned int i=0; i<visibleLayerBias.size(); i++){
469 file << visibleLayerBias[i];
470 if( i < visibleLayerBias.size()-1 ) file <<
" ";
474 file <<
"HiddenLayerBias: ";
475 for(
unsigned int i=0; i<hiddenLayerBias.size(); i++){
476 file << hiddenLayerBias[i];
477 if( i < hiddenLayerBias.size()-1 ) file <<
" ";
489 errorLog <<
"load(fstream &file) - The file is not open!" << std::endl;
498 if( word ==
"GRT_BERNOULLI_RBM_MODEL_FILE_V1.0" ){
502 if( word !=
"GRT_BERNOULLI_RBM_MODEL_FILE_V1.1" ){
503 errorLog <<
"load(fstream &file) - Failed to read file header!" << std::endl;
508 errorLog <<
"load(fstream &file) - Failed to load base settings to file!" << std::endl;
514 if( word !=
"NumVisibleUnits:" ){
515 errorLog <<
"load(fstream &file) - Failed to read NumVisibleUnits header!" << std::endl;
518 file >> numVisibleUnits;
522 if( word !=
"NumHiddenUnits:" ){
523 errorLog <<
"load(fstream &file) - Failed to read NumHiddenUnits header!" << std::endl;
526 file >> numHiddenUnits;
530 if( word !=
"BatchSize:" ){
531 errorLog <<
"load(fstream &file) - Failed to read BatchSize header!" << std::endl;
538 if( word !=
"BatchStepSize:" ){
539 errorLog <<
"load(fstream &file) - Failed to read BatchStepSize header!" << std::endl;
542 file >> batchStepSize;
546 if( word !=
"LearningRate:" ){
547 errorLog <<
"load(fstream &file) - Failed to read LearningRate header!" << std::endl;
550 file >> learningRate;
554 if( word !=
"LearningRateUpdate:" ){
555 errorLog <<
"load(fstream &file) - Failed to read LearningRateUpdate header!" << std::endl;
558 file >> learningRateUpdate;
562 if( word !=
"Momentum:" ){
563 errorLog <<
"load(fstream &file) - Failed to read Momentum header!" << std::endl;
570 if( word !=
"RandomizeWeightsForTraining:" ){
571 errorLog <<
"load(fstream &file) - Failed to read RandomizeWeightsForTraining header!" << std::endl;
574 file >> randomizeWeightsForTraining;
578 if( word !=
"Ranges:" ){
579 errorLog <<
"load(fstream &file) - Failed to read Ranges header!" << std::endl;
582 ranges.
resize(numInputDimensions);
583 for(UINT n=0; n<ranges.size(); n++){
584 file >> ranges[n].minValue;
585 file >> ranges[n].maxValue;
593 if( word !=
"WeightsMatrix:" ){
594 errorLog <<
"load(fstream &file) - Failed to read WeightsMatrix header!" << std::endl;
597 weightsMatrix.
resize(numHiddenUnits, numVisibleUnits);
599 for(UINT i=0; i<weightsMatrix.
getNumRows(); i++){
600 for(UINT j=0; j<weightsMatrix.
getNumCols(); j++){
601 file >> weightsMatrix[i][j];
607 if( word !=
"VisibleLayerBias:" ){
608 errorLog <<
"load(fstream &file) - Failed to read VisibleLayerBias header!" << std::endl;
611 visibleLayerBias.
resize(numVisibleUnits);
613 for(
unsigned int i=0; i<visibleLayerBias.size(); i++){
614 file >> visibleLayerBias[i];
619 if( word !=
"HiddenLayerBias:" ){
620 errorLog <<
"load(fstream &file) - Failed to read HiddenLayerBias header!" << std::endl;
623 hiddenLayerBias.
resize(numHiddenUnits);
625 for(
unsigned int i=0; i<hiddenLayerBias.size(); i++){
626 file >> hiddenLayerBias[i];
639 std::cout <<
"WeightsMatrix: \n";
640 for(UINT i=0; i<numVisibleUnits; i++) {
641 for(UINT j=0; j<numHiddenUnits; j++) {
642 std::cout << weightsMatrix[j][i] <<
"\t";
644 std::cout << std::endl;
646 std::cout << std::endl;
648 std::cout <<
"visible layer bias: ";
649 for(UINT j=0; j<numVisibleUnits; j++) {
650 std::cout << visibleLayerBias[j] <<
"\t";
652 std::cout << std::endl;
654 std::cout <<
"hidden layer bias: ";
655 for(UINT j=0; j<numHiddenUnits; j++) {
656 std::cout << hiddenLayerBias[j] <<
"\t";
658 std::cout << std::endl;
663 bool BernoulliRBM::getRandomizeWeightsForTraining()
const{
664 return randomizeWeightsForTraining;
667 UINT BernoulliRBM::getNumVisibleUnits()
const{
668 return numVisibleUnits;
671 UINT BernoulliRBM::getNumHiddenUnits()
const{
672 return numHiddenUnits;
675 const MatrixFloat& BernoulliRBM::getWeights()
const{
676 return weightsMatrix;
683 bool BernoulliRBM::setNumHiddenUnits(
const UINT numHiddenUnits){
684 this->numHiddenUnits = numHiddenUnits;
689 bool BernoulliRBM::setMomentum(
const Float momentum){
690 this->momentum = momentum;
694 bool BernoulliRBM::setLearningRateUpdate(
const Float learningRateUpdate){
695 this->learningRateUpdate = learningRateUpdate;
699 bool BernoulliRBM::setRandomizeWeightsForTraining(
const bool randomizeWeightsForTraining){
700 this->randomizeWeightsForTraining = randomizeWeightsForTraining;
704 bool BernoulliRBM::setBatchSize(
const UINT batchSize){
705 this->batchSize = batchSize;
709 bool BernoulliRBM::setBatchStepSize(
const UINT batchStepSize){
710 this->batchStepSize = batchStepSize;
717 UINT numGibbsSteps = 0;
720 errorLog <<
"load(fstream &file) - Failed to load base settings to file!" << std::endl;
726 if( word !=
"NumVisibleUnits:" ){
727 errorLog <<
"load(fstream &file) - Failed to read NumVisibleUnits header!" << std::endl;
730 file >> numVisibleUnits;
734 if( word !=
"NumHiddenUnits:" ){
735 errorLog <<
"load(fstream &file) - Failed to read NumHiddenUnits header!" << std::endl;
738 file >> numHiddenUnits;
742 if( word !=
"NumTrainingEpochs:" ){
743 errorLog <<
"load(fstream &file) - Failed to read NumTrainingEpochs header!" << std::endl;
746 file >> maxNumEpochs;
750 if( word !=
"NumGibbsSteps:" ){
751 errorLog <<
"load(fstream &file) - Failed to read NumGibbsSteps header!" << std::endl;
754 file >> numGibbsSteps;
758 if( word !=
"LearningRate:" ){
759 errorLog <<
"load(fstream &file) - Failed to read LearningRate header!" << std::endl;
762 file >> learningRate;
766 if( word !=
"LearningRateUpdate:" ){
767 errorLog <<
"load(fstream &file) - Failed to read LearningRateUpdate header!" << std::endl;
770 file >> learningRateUpdate;
774 if( word !=
"Momentum:" ){
775 errorLog <<
"load(fstream &file) - Failed to read Momentum header!" << std::endl;
782 if( word !=
"RandomizeWeightsForTraining:" ){
783 errorLog <<
"load(fstream &file) - Failed to read RandomizeWeightsForTraining header!" << std::endl;
786 file >> randomizeWeightsForTraining;
790 if( word !=
"Ranges:" ){
791 errorLog <<
"load(fstream &file) - Failed to read Ranges header!" << std::endl;
794 ranges.
resize(numInputDimensions);
795 for(UINT n=0; n<ranges.size(); n++){
796 file >> ranges[n].minValue;
797 file >> ranges[n].maxValue;
805 if( word !=
"WeightsMatrix:" ){
806 errorLog <<
"load(fstream &file) - Failed to read WeightsMatrix header!" << std::endl;
809 weightsMatrix.
resize(numHiddenUnits, numVisibleUnits);
811 for(UINT i=0; i<weightsMatrix.
getNumRows(); i++){
812 for(UINT j=0; j<weightsMatrix.
getNumCols(); j++){
813 file >> weightsMatrix[i][j];
819 if( word !=
"VisibleLayerBias:" ){
820 errorLog <<
"load(fstream &file) - Failed to read VisibleLayerBias header!" << std::endl;
823 visibleLayerBias.
resize(numVisibleUnits);
825 for(
unsigned int i=0; i<visibleLayerBias.
getSize(); i++){
826 file >> visibleLayerBias[i];
831 if( word !=
"HiddenLayerBias:" ){
832 errorLog <<
"load(fstream &file) - Failed to read HiddenLayerBias header!" << std::endl;
835 hiddenLayerBias.
resize(numHiddenUnits);
837 for(
unsigned int i=0; i<hiddenLayerBias.
getSize(); i++){
838 file >> hiddenLayerBias[i];
bool saveBaseSettingsToFile(std::fstream &file) const
virtual bool save(std::fstream &file) const
bool predict_(VectorFloat &inputData)
virtual bool resize(const unsigned int size)
This class implements a Bernoulli Restricted Boltzmann machine.
signed long getMilliSeconds()
bool setAllValues(const T &value)
bool add(const MatrixFloat &b)
bool subtract(const MatrixFloat &b)
unsigned int getNumRows() const
bool loadLegacyModelFromFile(std::fstream &file)
unsigned int getNumCols() const
bool loadBaseSettingsFromFile(std::fstream &file)
T ** getDataPointer() const
virtual bool print() const
Vector< MinMax > getRanges() const
virtual bool load(std::fstream &file)
Float getRandomNumberUniform(Float minRange=0.0, Float maxRange=1.0)
virtual bool train_(MatrixFloat &data)
virtual bool resize(const unsigned int r, const unsigned int c)
MatrixFloat multiple(const Float value) const