fn_crossvalidation.hpp

Go to the documentation of this file.

// Copyright (C) 2011 the authors listed below
// http://ecocpak.sourceforge.net
//
// Authors:
// - Dimitrios Bouzas (bouzas at ieee dot org)
// - Nikolaos Arvanitopoulos (niarvani at ieee dot org)
// - Anastasios Tefas (tefas at aiia dot csd dot auth dot gr)
//
// This file is part of the ECOC PAK C++ library. It is
// provided without any warranty of fitness for any purpose.
//
// You can redistribute this file and/or modify it under
// the terms of the GNU Lesser General Public License (LGPL)
// as published by the Free Software Foundation, either
// version 3 of the License or (at your option) any later
// version.
// (see http://www.opensource.org/licenses for more info)





double
cross_validation
  (
  const mat& samples,
  const icolvec& labels,
  const imat& codmat,
  const Threshold& thres,
  const u32 n_folds,
  const u32 n_classes,
  const u32 criterion_option,
  const u32 classifier_option,
  const int coding_strategy,
  const int decoding_strategy,
  const u32 n_matrices,
  const u32 n_desired_classifers,
  const double validation,
  const int init_coding_strategy,
  const int ecocone_mode,
  const u32 max_iter,
  const double epsilon,
  const double wvalidation,
  const bool verbose,
  double& mean_rows,
  double& mean_cols,
  double& elapsed_time,
  ofstream& output
  )
  {
  // initialize
  mean_rows = 0.0;
  mean_cols = 0.0;
  elapsed_time = 0.0;
  double tmp_time = 0.0;

  double *target = Malloc(double, n_folds);

  int i;
  int* fold_start = Malloc(int, n_folds + 1);
  int l = samples.n_rows;
  int* perm = Malloc(int, l);
  int nr_class;

  // stratified cv may not give leave-one-out rate
  // Each class to l folds -> some folds may have zero elements
  if(n_folds < l)
    {
    int* start = NULL;
    int* label = NULL;
    int* count = NULL;
    group_classes(labels, &nr_class, &label, &start, &count, perm);

    // random shuffle and then data grouped by fold using the array perm
    int* fold_count = Malloc(int, n_folds);
    int c;
    int* index = Malloc(int, l);

    for(i = 0; i < l; i++)
      {
      index[i] = perm[i];
      }

    for(c = 0; c < nr_class; c++)
      {

      for(i = 0;i < count[c]; i++)
        {
        int j = i + rand() % (count[c] - i);
        swap(index[start[c] + j], index[start[c] + i]);
        }

      }

    for(i = 0; i < n_folds; i++)
      {
      fold_count[i] = 0;

      for(c = 0; c < nr_class; c++)
        {
        fold_count[i] +=
          (i + 1) * count[c] / n_folds - i * count[c] / n_folds;
        }

      }

    fold_start[0] = 0;
    for(i = 1; i <= n_folds; i++)
      {
      fold_start[i] = fold_start[i - 1] + fold_count[i - 1];
      }

    for(c = 0; c < nr_class; c++)
      {
      for(i = 0; i < n_folds; i++)
        {
        int begin = start[c] + i * count[c] / n_folds;
        int end = start[c] + (i + 1) * count[c] / n_folds;

        for(int j = begin; j < end; j++)
          {
          perm[fold_start[i]] = index[j];
          fold_start[i]++;
          }

        }

      }

    fold_start[0] = 0;

    for(i = 1; i <= n_folds; i++)
      {
      fold_start[i] = fold_start[i - 1] + fold_count[i - 1];
      }

    free(start);
    free(label);
    free(count);
    free(index);
    free(fold_count);
    }
  else
    {
    for(i = 0; i < l; i++)
      {
      perm[i] = i;
      }

    for(i = 0; i < l; i++)
      {
      int j = i + rand() % (l - i);
      swap(perm[i], perm[j]);
      }

    for(i = 0; i <= n_folds; i++)
      {
      fold_start[i] = i * l / n_folds;
      }

    }

  // for each fold
  for(i = 0; i < n_folds; i++)
    {
    #ifdef NCURSES_OUTPUT
    if(menu_on == true)
      {
      clear();
      mvprintw(0, 0, "Running FOLD NR. %d\n", i + 1);
      refresh();
      }
    #endif

    if(verbose == true)
      {
      output << endl << ">> Fold Number " <<  i + 1 << endl << endl;
      }

    int begin = fold_start[i];
    int end = fold_start[i + 1];
    int j;
    int k;

    // --- costruct the sub problem that corresponds to the       --- //
    // --- specific fold                                          --- //

    // number of training samples
    const u32 training_size = l - (end - begin);

    // number of test samples
    const u32 test_size = end - begin;

    // allocate memory for subdata matrix and sublabels
    mat subDataMat = zeros<mat>(training_size, samples.n_cols);
    icolvec subLabels = zeros<icolvec>(training_size);

    // create the subproblem by assigning the correct samples and labels
    // according to the created permutations from the cross validation
    // procedure
    k = 0;
    for(j = 0; j < begin; j++)
      {
      subDataMat.row(k) = samples.row(perm[j]);
      subLabels(k) = labels(perm[j]);

      ++k;
      }

    for(j = end; j < l; j++)
      {
      subDataMat.row(k) = samples.row(perm[j]);
      subLabels(k) = labels(perm[j]);

      ++k;
      }

    // --- end of subproblem construction --- //

    // --- costruct the test subproblem that corresponds to the   --- //
    // --- specific fold                                          --- //

    // allocate memory for testLabels
    icolvec test_labels = zeros<icolvec>(test_size);

    // allocate test set
    mat test_set = zeros<mat>(test_size, samples.n_cols);

    // create the test subproblem and labels
    k = 0;
    for(j = begin; j < end; j++)
      {
      test_labels(k) = labels(perm[j]);
      test_set.row(k) = samples.row(perm[j]);
      ++k;
      }

    // current fold coding matrix
    imat coding_matrix;

    // current fold vector of classifiers
    vector<Classifier*> classifiers_vector;

    // --- end of test subproblem construction --- //
    switch(coding_strategy)
      {
      // Discriminant Error Correcting Output Codes Coding
      case DECOC:
        {
        target[i] = decoc
                      (
                      subDataMat,
                      subLabels,
                      test_set,
                      test_labels,
                      decoding_strategy,
                      classifier_option,
                      criterion_option,
                      verbose,
                      output,
                      tmp_time
                      );

        elapsed_time += tmp_time;

        break;
        }

      // Discriminant Error Correcting Output Codes with subclasses
      case SUBDECOC:
        {
        target[i] = subdecoc
                      (
                      subDataMat,
                      subLabels,
                      test_set,
                      test_labels,
                      thres,
                      decoding_strategy,
                      classifier_option,
                      criterion_option,
                      verbose,
                      coding_matrix,
                      output,
                      tmp_time
                      );

        mean_rows += double(coding_matrix.n_rows) / double(n_folds);
        mean_cols += double(coding_matrix.n_cols) / double(n_folds);
        elapsed_time += tmp_time;

        break;
        }

      // One versus One, or all pairs coding
      case ONE_VS_ONE:
        {
        target[i] = one_vs_one
                      (
                      subDataMat,
                      subLabels,
                      test_set,
                      test_labels,
                      decoding_strategy,
                      classifier_option,
                      verbose,
                      output,
                      tmp_time
                      );

        elapsed_time += tmp_time;

        break;
        }

      // One versus All, or one against all coding
      case ONE_VS_ALL:
        {
        target[i] = one_vs_all
                      (
                      subDataMat,
                      subLabels,
                      test_set,
                      test_labels,
                      decoding_strategy,
                      classifier_option,
                      verbose,
                      output,
                      tmp_time
                      );

        elapsed_time += tmp_time;

        break;
        }

      // Dense Random coding
      case DENSE_RANDOM:
        {
        target[i] = dense_random_ecoc
                      (
                      subDataMat,
                      subLabels,
                      test_set,
                      test_labels,
                      decoding_strategy,
                      classifier_option,
                      n_matrices,
                      n_desired_classifers,
                      verbose,
                      output,
                      tmp_time
                      );

        elapsed_time += tmp_time;

        break;
        }

      // Sparse Random coding
      case SPARSE_RANDOM:
        {
        target[i] = sparse_random_ecoc
                      (
                      subDataMat,
                      subLabels,
                      test_set,
                      test_labels,
                      decoding_strategy,
                      classifier_option,
                      n_matrices,
                      n_desired_classifers,
                      verbose,
                      output,
                      tmp_time
                      );

        elapsed_time += tmp_time;

        break;
        }

      // ECOC One
      case ECOC_ONE:
        {
        target[i] = ecoc_one
                      (
                      subDataMat,
                      subLabels,
                      test_set,
                      test_labels,
                      thres,
                      decoding_strategy,
                      classifier_option,
                      criterion_option,
                      n_matrices,
                      n_desired_classifers,
                      validation,
                      init_coding_strategy,
                      ecocone_mode,
                      max_iter,
                      epsilon,
                      wvalidation,
                      verbose,
                      output,
                      tmp_time
                      );

        elapsed_time += tmp_time;

        break;
        }

      // Forest ECOC
      case FOREST_ECOC:
        {
        target[i] = forest_ecoc
                      (
                      subDataMat,
                      subLabels,
                      test_set,
                      test_labels,
                      decoding_strategy,
                      classifier_option,
                      criterion_option,
                      n_matrices,
                      verbose,
                      output,
                      tmp_time
                      );

        elapsed_time += tmp_time;

        break;
        }

      // User Custom coding
      case CUSTOM_CODING:
        {
        target[i] = custom_coding
                      (
                      subDataMat,
                      subLabels,
                      test_set,
                      test_labels,
                      codmat,
                      decoding_strategy,
                      classifier_option,
                      verbose,
                      output,
                      tmp_time
                      );

        elapsed_time += tmp_time;

        break;
        }

      }

    target[i] /= double(test_set.n_rows);

    ClassData::globalIndex = 0;
    }

  free(fold_start);
  free(perm);

  double tempSum = 0.0;
  for(u32 i = 0; i < n_folds; i++)
    {
    tempSum += target[i];
    }

  tempSum /= double(n_folds);

  free(target);

  return tempSum;
  }



double
cvonesvsone
  (
  const mat& samples,
  const icolvec& labels,
  const u32 n_folds,
  const u32 n_classes,
  const u32 classifier_option,
  const int decoding_strategy,
  const bool verbose,
  double& elapsed_time,
  ofstream& output
  )
  {
  // dummies, overrides
  imat codmat;
  Threshold thres(100,100,100);
  double mr;
  double mc;

  return cross_validation
           (
           samples,
           labels,
           codmat,
           thres,
           n_folds,
           n_classes,
           0,
           classifier_option,
           ONE_VS_ONE,
           decoding_strategy,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           verbose,
           mr,
           mc,
           elapsed_time,
           output
           );

  }



double
cvonesvsall
  (
  const mat& samples,
  const icolvec& labels,
  const u32 n_folds,
  const u32 n_classes,
  const u32 classifier_option,
  const int decoding_strategy,
  const bool verbose,
  double& elapsed_time,
  ofstream& output
  )
  {
  // dummies, overrides
  imat codmat;
  Threshold thres(100,100,100);
  double mr;
  double mc;

  return cross_validation
           (
           samples,
           labels,
           codmat,
           thres,
           n_folds,
           n_classes,
           0,
           classifier_option,
           ONE_VS_ALL,
           decoding_strategy,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           verbose,
           mr,
           mc,
           elapsed_time,
           output
           );

  }



double
cvdecoc
  (
  const mat& samples,
  const icolvec& labels,
  const u32 n_folds,
  const u32 n_classes,
  const u32 classifier_option,
  const int decoding_strategy,
  const u32 criterion_option,
  const bool verbose,
  double& elapsed_time,
  ofstream& output
  )
  {
  // dummies, overrides
  imat codmat;
  Threshold thres(100,100,100);
  double mr;
  double mc;

  return cross_validation
           (
           samples,
           labels,
           codmat,
           thres,
           n_folds,
           n_classes,
           criterion_option,
           classifier_option,
           DECOC,
           decoding_strategy,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           verbose,
           mr,
           mc,
           elapsed_time,
           output
           );

  }



double
cvsubdecoc
  (
  const mat& samples,
  const icolvec& labels,
  const u32 n_folds,
  const u32 n_classes,
  const u32 classifier_option,
  const int decoding_strategy,
  const u32 criterion_option,
  const Threshold& thres,
  const bool verbose,
  double& mr,
  double& mc,
  double& elapsed_time,
  ofstream& output
  )
  {
  // dummies, overrides
  imat codmat;

  return cross_validation
           (
           samples,
           labels,
           codmat,
           thres,
           n_folds,
           n_classes,
           criterion_option,
           classifier_option,
           SUBDECOC,
           decoding_strategy,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           verbose,
           mr,
           mc,
           elapsed_time,
           output
           );

  }



double
cvsparserand
  (
  const mat& samples,
  const icolvec& labels,
  const u32 n_folds,
  const u32 n_classes,
  const u32 classifier_option,
  const int decoding_strategy,
  const u32 n_matrices,
  const u32 n_desired_classifers,
  const bool verbose,
  double& elapsed_time,
  ofstream& output
  )
  {
  // dummies, overrides
  imat codmat;
  Threshold thres(100,100,100);
  double mr;
  double mc;

  return cross_validation
           (
           samples,
           labels,
           codmat,
           thres,
           n_folds,
           n_classes,
           0,
           classifier_option,
           SPARSE_RANDOM,
           decoding_strategy,
           n_matrices,
           n_desired_classifers,
           0,
           0,
           0,
           0,
           0,
           0,
           verbose,
           mr,
           mc,
           elapsed_time,
           output
           );

  }



double
cvdenserand
  (
  const mat& samples,
  const icolvec& labels,
  const u32 n_folds,
  const u32 n_classes,
  const u32 classifier_option,
  const int decoding_strategy,
  const u32 n_matrices,
  const u32 n_desired_classifers,
  const bool verbose,
  double& elapsed_time,
  ofstream& output
  )
  {
  // dummies, overrides
  imat codmat;
  Threshold thres(100,100,100);
  double mr;
  double mc;

  return cross_validation
           (
           samples,
           labels,
           codmat,
           thres,
           n_folds,
           n_classes,
           0,
           classifier_option,
           DENSE_RANDOM,
           decoding_strategy,
           n_matrices,
           n_desired_classifers,
           0,
           0,
           0,
           0,
           0,
           0,
           verbose,
           mr,
           mc,
           elapsed_time,
           output
           );

  }



double
cvecocone
  (
  const mat& samples,
  const icolvec& labels,
  const u32 n_folds,
  const u32 n_classes,
  const u32 classifier_option,
  const int decoding_strategy,
  const double validation,
  const int init_coding_strategy,
  const int ecocone_mode,
  const u32 max_iter,
  const double epsilon,
  const double wvalidation,
  const bool verbose,
  double& elapsed_time,
  ofstream& output
  )
  {
  // dummies, overrides
  imat codmat;
  Threshold thres(100,100,100);
  double mr;
  double mc;

  return cross_validation
           (
           samples,
           labels,
           codmat,
           thres,
           n_folds,
           n_classes,
           0,
           classifier_option,
           ECOC_ONE,
           decoding_strategy,
           0,
           0,
           validation,
           init_coding_strategy,
           ecocone_mode,
           max_iter,
           epsilon,
           wvalidation,
           verbose,
           mr,
           mc,
           elapsed_time,
           output
           );

  }



double
cvforest
  (
  const mat& samples,
  const icolvec& labels,
  const u32 n_folds,
  const u32 n_classes,
  const u32 classifier_option,
  const int decoding_strategy,
  const u32 n_trees,
  const bool verbose,
  double& elapsed_time,
  ofstream& output
  )
  {
  // dummies, overrides
  imat codmat;
  Threshold thres(100,100,100);
  double mr;
  double mc;

  return cross_validation
           (
           samples,
           labels,
           codmat,
           thres,
           n_folds,
           n_classes,
           0,
           classifier_option,
           FOREST_ECOC,
           decoding_strategy,
           n_trees,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           verbose,
           mr,
           mc,
           elapsed_time,
           output
           );

  }



double
cvcustom
  (
  const mat& samples,
  const icolvec& labels,
  const u32 n_folds,
  const u32 n_classes,
  const u32 classifier_option,
  const int decoding_strategy,
  const imat& codmat,
  const bool verbose,
  double& elapsed_time,
  ofstream& output
  )
  {
  // dummies, overrides
  Threshold thres(100,100,100);
  double mr;
  double mc;

  return cross_validation
           (
           samples,
           labels,
           codmat,
           thres,
           n_folds,
           n_classes,
           0,
           classifier_option,
           CUSTOM_CODING,
           decoding_strategy,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           verbose,
           mr,
           mc,
           elapsed_time,
           output
           );

  }