00001
00005 #include <assert.h>
00006 #include <cmath>
00007 #include <algorithm>
00008 #include <set>
00009 #include "multiclass_ecoc.h"
00010
00011 REGISTER_CREATOR(lemga::MultiClass_ECOC);
00012
00013 #define LABEL_EQUAL(x,y) (std::fabs((x)-(y)) < EPSILON)
00014
00015 #define LABEL2INDEX(y) \
00016 std::lower_bound(labels.begin(), labels.end(), y) - labels.begin()
00017
00018 namespace lemga {
00019
00020 bool MultiClass_ECOC::serialize (std::ostream& os, ver_list& vl) const {
00021 SERIALIZE_PARENT(Aggregating, os, vl, 1);
00022 const UINT n = lm.size();
00023
00024 assert(lm_wgt.size() == n);
00025 for (UINT i = 0; i < n; ++i)
00026 os << lm_wgt[i] << ' ';
00027 if (n) os << '\n';
00028
00029 os << ecoc.size() << '\n';
00030 for (UINT i = 0; i < ecoc.size(); ++i) {
00031 assert(n <= ecoc[i].size());
00032 for (UINT j = 0; j < n; ++j)
00033 os << ecoc[i][j] << ' ';
00034 if (n) os << '\n';
00035 }
00036
00037 os << nclass << '\n';
00038 assert(nclass == labels.size());
00039 for (UINT i = 0; i < nclass; ++i)
00040 os << labels[i] << ' ';
00041 if (nclass) os << '\n';
00042 return os;
00043 }
00044
00045 bool
00046 MultiClass_ECOC::unserialize (std::istream& is, ver_list& vl, const id_t& d) {
00047 if (d != id() && d != NIL_ID) return false;
00048 UNSERIALIZE_PARENT(Aggregating, is, vl, 1, v);
00049
00050 const UINT n = lm.size();
00051 lm_wgt.resize(n);
00052 for (UINT i = 0; i < n; ++i)
00053 if (!(is >> lm_wgt[i])) return false;
00054
00055 UINT c;
00056 if (!(is >> c)) return false;
00057 ecoc.resize(c);
00058 for (UINT i = 0; i < c; ++i) {
00059 ecoc[i].resize(n);
00060 for (UINT j = 0; j < n; ++j)
00061 if (!(is >> ecoc[i][j])) return false;
00062 }
00063
00064 if (!(is >> nclass)) return false;
00065 labels.resize(nclass);
00066 for (UINT i = 0; i < nclass; ++i)
00067 if (!(is >> labels[i])) return false;
00068
00069 return true;
00070 }
00071
00072 void MultiClass_ECOC::set_ECOC_table (const ECOC_TABLE& e) {
00073 assert(!e.empty());
00074 ecoc = e;
00075 if (max_n_model == 0)
00076 set_max_models(ecoc[0].size());
00077 }
00078
00079 void MultiClass_ECOC::set_ECOC_table (UINT i, const ECOC_VECTOR& p) {
00080 for (UINT c = 0; c < p.size(); ++c) {
00081 if (ecoc[c].size() <= i) {
00082 assert(ecoc[c].size() == i);
00083 ecoc[c].push_back(p[c]);
00084 }
00085 else ecoc[c][i] = p[c];
00086 }
00087 }
00088
00089 void MultiClass_ECOC::set_ECOC_table (ECOC_TYPE et) {
00090 assert(et != NO_TYPE);
00091 if (nclass == 0) {
00092 ecoc_type = et;
00093 return;
00094 }
00095
00096 ECOC_TABLE e;
00097
00098 switch (et) {
00099 case ONE_VS_ONE: {
00100 UINT m = nclass*(nclass-1)/2;
00101 e = ECOC_TABLE(nclass, ECOC_VECTOR(m, 0));
00102 UINT k = 0;
00103 for (UINT i = 0; i < nclass; ++i)
00104 for (UINT j = i+1; j < nclass; ++j, ++k) {
00105 e[i][k] = 1; e[j][k] = -1;
00106 }
00107 }
00108 break;
00109
00110 case ONE_VS_ALL:
00111 e = ECOC_TABLE(nclass, ECOC_VECTOR(nclass, -1));
00112 for (UINT i = 0; i < nclass; ++i)
00113 e[i][i] = 1;
00114 break;
00115
00116 default:
00117 assert(false);
00118 }
00119
00120 set_ECOC_table(e);
00121 }
00122
00123
00124 REAL MultiClass_ECOC::c_error (const Output& out, const Output& y) const {
00125 assert(n_output() == 1);
00126 return !LABEL_EQUAL(out[0], y[0]);
00127 }
00128
00129 void MultiClass_ECOC::reset () {
00130 Aggregating::reset();
00131 lm_wgt.clear();
00133 ecoc.clear(); ecoc_type = NO_TYPE;
00134 #if MULTICLASS_ECOC_OUTPUT_CACHE
00135 if (nclass > 0) clear_cache();
00136 #endif
00137 }
00138
00139 void MultiClass_ECOC::set_train_data (const pDataSet& pd,
00140 const pDataWgt& pw) {
00141 pDataSet old_ptd = ptd;
00142 Aggregating::set_train_data(pd, pw);
00143 if (old_ptd == ptd) {
00144 assert(ecoc_type == NO_TYPE);
00145 return;
00146 }
00147
00148 std::set<REAL> ls;
00149 for (UINT i = 0; i < n_samples; ++i)
00150 ls.insert(ptd->y(i)[0]);
00151
00152
00153 std::vector<REAL> new_labels(ls.begin(), ls.end());
00154 if (nclass > 0) {
00155 bool same = (nclass == new_labels.size());
00156 for (UINT c = 0; same && c < nclass; ++c)
00157 same = LABEL_EQUAL(labels[c], new_labels[c]);
00158 if (!same)
00159 std::cerr << "MultiClass_ECOC: Warning: "
00160 "Class label mapping is changed\n";
00161 }
00162
00163
00164 labels.swap(new_labels);
00165 nclass = labels.size();
00166 for (UINT c = 1; c < nclass; ++c)
00167 if (LABEL_EQUAL(labels[c-1], labels[c])) {
00168 std::cerr << "MultiClass_ECOC: Error: Labels ("
00169 << labels[c-1] << " and " << labels[c]
00170 << ") are too close. Alter LABEL_EQUAL?\n";
00171 std::exit(-1);
00172 }
00173
00174
00175 ex_class.resize(n_samples);
00176 for (UINT i = 0; i < n_samples; ++i) {
00177 REAL y = ptd->y(i)[0];
00178 UINT c = LABEL2INDEX(y);
00179 assert(c < nclass && LABEL_EQUAL(y, labels[c]));
00180 ex_class[i] = c;
00181 }
00182
00183 if (ecoc_type != NO_TYPE) {
00184 set_ECOC_table(ecoc_type);
00185 ecoc_type = NO_TYPE;
00186 } else
00187 ecoc.resize(nclass);
00188 #if MULTICLASS_ECOC_OUTPUT_CACHE
00189 clear_cache();
00190 #endif
00191 local_d.resize(nclass);
00192 }
00193
00194 void MultiClass_ECOC::train () {
00195 assert(ptd != 0 && ptw != 0);
00196 assert(lm_base != 0);
00197 assert(_n_out == 1);
00198 assert(nclass <= ecoc.size() && nclass == labels.size());
00199
00200
00201 n_in_agg = lm.size();
00202 assert(n_in_agg == lm_wgt.size());
00203 setup_aux();
00204
00205 while (n_in_agg < max_n_model) {
00206 ECOC_VECTOR par;
00207 if (!ECOC_partition(n_in_agg, par)) break;
00208 #if VERBOSE_OUTPUT
00209 std::cout << "*** " << id() << " #" << n_in_agg+1
00210 << " / " << max_n_model << " ***\n";
00211 #endif
00212
00213 const pLearnModel p = train_with_partition(par);
00214
00215 const REAL w = assign_weight(par, *p);
00216 if (w <= 0) break;
00217
00218 set_dimensions(*p);
00219 lm.push_back(p); lm_wgt.push_back(w);
00220 set_ECOC_table(n_in_agg, par);
00221 ++n_in_agg;
00222 update_aux(par);
00223 }
00224 }
00225
00226
00227 #define GET_BEST_CLASS(distance_to_class_c) \
00228 REAL dmin = INFINITY; UINT cmin = UINT(-1); \
00229 for (UINT c = 0; c < nclass; ++c) { \
00230 REAL dc = distance_to_class_c; \
00231 assert(dc < INFINITY/10); \
00232 if (dc < dmin) { dmin = dc; cmin = c; } \
00233 }
00234 #define GET_MARGIN(distance_to_class_c,y) \
00235 REAL dy = 0, dmin = INFINITY; \
00236 for (UINT c = 0; c < nclass; ++c) { \
00237 REAL dc = distance_to_class_c; \
00238 assert(dc < INFINITY/10); \
00239 if (c == y) dy = dc; \
00240 else if (dc < dmin) dmin = dc; \
00241 }
00242
00243 const std::vector<REAL>& MultiClass_ECOC::distances (const Input& x) const {
00244 assert(n_in_agg <= lm.size() && n_in_agg <= lm_wgt.size());
00245 #ifndef NDEBUG
00246 for (UINT i = 0; i < n_in_agg; ++i)
00247 assert(lm_wgt[i] >= 0);
00248 #endif
00249
00250 assert(n_output() == 1);
00251 Output out(n_in_agg);
00252 for (UINT i = 0; i < n_in_agg; ++i) {
00253 assert(lm[i] != 0);
00254 out[i] = (*lm[i])(x)[0];
00255 }
00256 std::vector<REAL>& d = local_d;
00257 assert(d.size() == nclass);
00258 for (UINT c = 0; c < nclass; ++c)
00259 d[c] = ECOC_distance(out, ecoc[c]);
00260 return d;
00261 }
00262
00263 const std::vector<REAL>& MultiClass_ECOC::distances (UINT idx) const {
00264 #if MULTICLASS_ECOC_OUTPUT_CACHE == 2 // distance cache
00265 assert(cache_n.size() == n_samples);
00266 if (cache_n[idx] > n_in_agg)
00267 clear_cache(idx);
00268 std::vector<REAL>& d = cache_d[idx];
00269 assert(d.size() == nclass);
00270 for (UINT i = cache_n[idx]; i < n_in_agg; ++i) {
00271 REAL o = lm[i]->get_output(idx)[0];
00272 for (UINT c = 0; c < nclass; ++c)
00273 d[c] = ECOC_distance(o, ecoc[c][i], lm_wgt[i], d[c]);
00274 }
00275 cache_n[idx] = n_in_agg;
00276 #else // compute the output first
00277 #if MULTICLASS_ECOC_OUTPUT_CACHE == 1
00278 assert(cache_o.size() == n_samples);
00279 Output& out = cache_o[idx];
00280 for (UINT i = out.size(); i < n_in_agg; ++i) {
00281 assert(lm[i] != 0);
00282 out.push_back(lm[i]->get_output(idx)[0]);
00283 }
00284 #elif !MULTICLASS_ECOC_OUTPUT_CACHE
00285 Output out(n_in_agg);
00286 for (UINT i = 0; i < n_in_agg; ++i) {
00287 assert(lm[i] != 0);
00288 out[i] = lm[i]->get_output(idx)[0];
00289 }
00290 #else
00291 #error "Wrong value of MULTICLASS_ECOC_OUTPUT_CACHE"
00292 #endif
00293 std::vector<REAL>& d = local_d;
00294 assert(d.size() == nclass);
00295 for (UINT c = 0; c < nclass; ++c)
00296 d[c] = ECOC_distance(out, ecoc[c]);
00297 #endif
00298 return d;
00299 }
00300
00301 Output MultiClass_ECOC::operator() (const Input& x) const {
00302 const std::vector<REAL>& d = distances(x);
00303 GET_BEST_CLASS(d[c]);
00304 return Output(1, labels[cmin]);
00305 }
00306
00307 Output MultiClass_ECOC::get_output (UINT idx) const {
00308 assert(ptw != 0);
00309 assert(nclass <= ecoc.size() && nclass == labels.size());
00310
00311 const std::vector<REAL>& d = distances(idx);
00312 GET_BEST_CLASS(d[c]);
00313
00314
00315
00316
00317 return Output(1, labels[cmin]);
00318 }
00319
00320 REAL MultiClass_ECOC::margin_of (const Input& x, const Output& l) const {
00321 UINT y = LABEL2INDEX(l[0]);
00322 assert(y < nclass && LABEL_EQUAL(l[0], labels[y]));
00323 const std::vector<REAL>& d = distances(x);
00324 GET_MARGIN(d[c], y);
00325 return dmin - dy;
00326 }
00327
00328 REAL MultiClass_ECOC::margin (UINT idx) const {
00329 assert(ptw != 0);
00330 UINT y = ex_class[idx];
00331
00332 const std::vector<REAL>& d = distances(idx);
00333 GET_MARGIN(d[c], y);
00334 return dmin - dy;
00335 }
00336
00337 REAL MultiClass_ECOC::ECOC_distance (const Output& o,
00338 const ECOC_VECTOR& cw) const {
00339 assert(n_in_agg <= o.size() && n_in_agg <= cw.size());
00340 assert(n_in_agg <= lm_wgt.size());
00341 REAL ip = 0;
00342 for (UINT i = 0; i < n_in_agg; ++i)
00343 ip += lm_wgt[i] * (o[i]>0? -.5:.5) * cw[i];
00344 return ip;
00345 }
00346
00347 #if MULTICLASS_ECOC_OUTPUT_CACHE == 2 // distance cache
00348 REAL MultiClass_ECOC::ECOC_distance (REAL o, int cw, REAL w, REAL d) const {
00349 return d + w * (o>0? -.5:.5) * cw;
00350 }
00351 #endif
00352
00353 bool MultiClass_ECOC::is_full_partition (const ECOC_VECTOR& p) const {
00354 assert(p.size() == nclass);
00355 for (UINT c = 0; c < nclass; ++c)
00356 if (p[c] != -1 && p[c] != 1) return false;
00357 return true;
00358 }
00359
00360 bool MultiClass_ECOC::ECOC_partition (UINT i, ECOC_VECTOR& p) const {
00361 assert(nclass <= ecoc.size());
00362 p.resize(nclass);
00363 for (UINT c = 0; c < nclass; ++c) {
00364 if (i >= ecoc[c].size()) return false;
00365 p[c] = ecoc[c][i];
00366 }
00367 return true;
00368 }
00369
00370 pLearnModel MultiClass_ECOC::train_with_partition (ECOC_VECTOR& p) const {
00371 LearnModel *plm = lm_base->clone();
00372 assert(plm != 0);
00373
00374
00375 DataSet* btd = new DataSet();
00376 DataWgt* btw = new DataWgt();
00377 REAL wsum = 0;
00378 for (UINT i = 0; i < n_samples; ++i) {
00379 const Input& x = ptd->x(i);
00380 int b = p[ex_class[i]];
00381 if (b != 0) {
00382 assert(b == 1 || b == -1);
00383 btd->append(x, Output(1, b));
00384 btw->push_back((*ptw)[i]);
00385 wsum += (*ptw)[i];
00386 }
00387 }
00388 assert(wsum > 0);
00389 for (UINT i = 0; i < btw->size(); ++i)
00390 (*btw)[i] /= wsum;
00391
00392 plm->set_train_data(btd, btw);
00393 plm->train();
00394
00395 plm->set_train_data(ptd, ptw);
00396 return plm;
00397 }
00398
00399 REAL MultiClass_ECOC::cost () const {
00400 assert(ptd != 0 && ptw != 0);
00401 REAL cst = 0;
00402 for (UINT i = 0; i < n_samples; ++i) {
00403 const std::vector<REAL>& d = distances(i);
00404 const UINT y = ex_class[i];
00405 REAL dy = d[y], csti = 0;
00406 for (UINT c = 0; c < nclass; ++c)
00407 if (c != y) {
00408 csti += std::exp(dy - d[c]);
00409 }
00410 cst += (*ptw)[i] * csti;
00411 }
00412 return cst;
00413 }
00414
00415 }