00001
00005 #include <assert.h>
00006 #include <cmath>
00007 #include <set>
00008 #include "multiclass_ecoc.h"
00009
00010 REGISTER_CREATOR(lemga::MultiClass_ECOC);
00011
00012 #define LABEL_EQUAL(x,y) (std::fabs((x)-(y)) < EPSILON)
00013
00014 #define LABEL2INDEX(y) \
00015 std::lower_bound(labels.begin(), labels.end(), y) - labels.begin()
00016
00017 namespace lemga {
00018
00019 bool MultiClass_ECOC::serialize (std::ostream& os, ver_list& vl) const {
00020 SERIALIZE_PARENT(Aggregating, os, vl, 1);
00021 const UINT n = lm.size();
00022
00023 assert(lm_wgt.size() == n);
00024 for (UINT i = 0; i < n; ++i)
00025 os << lm_wgt[i] << ' ';
00026 if (n) os << '\n';
00027
00028 os << ecoc.size() << '\n';
00029 for (UINT i = 0; i < ecoc.size(); ++i) {
00030 assert(n <= ecoc[i].size());
00031 for (UINT j = 0; j < n; ++j)
00032 os << ecoc[i][j] << ' ';
00033 if (n) os << '\n';
00034 }
00035
00036 os << nclass << '\n';
00037 assert(nclass == labels.size());
00038 for (UINT i = 0; i < nclass; ++i)
00039 os << labels[i] << ' ';
00040 if (nclass) os << '\n';
00041 return os;
00042 }
00043
00044 bool MultiClass_ECOC::unserialize (std::istream& is, ver_list& vl,
00045 const id_t& d) {
00046 if (d != id() && d != empty_id) return false;
00047 UNSERIALIZE_PARENT(Aggregating, is, vl, 1, v);
00048
00049 const UINT n = lm.size();
00050 lm_wgt.resize(n);
00051 for (UINT i = 0; i < n; ++i)
00052 if (!(is >> lm_wgt[i])) return false;
00053
00054 UINT c;
00055 if (!(is >> c)) return false;
00056 ecoc.resize(c);
00057 for (UINT i = 0; i < c; ++i) {
00058 ecoc[i].resize(n);
00059 for (UINT j = 0; j < n; ++j)
00060 if (!(is >> ecoc[i][j])) return false;
00061 }
00062
00063 if (!(is >> nclass)) return false;
00064 labels.resize(nclass);
00065 for (UINT i = 0; i < nclass; ++i)
00066 if (!(is >> labels[i])) return false;
00067
00068 return true;
00069 }
00070
00071 void MultiClass_ECOC::set_ECOC_table (const ECOC_TABLE& e) {
00072 assert(!e.empty());
00073 ecoc = e;
00074 UINT n = ecoc[0].size();
00075 if (max_n_model < n)
00076 set_max_models(n);
00077
00078 }
00079
00080 void MultiClass_ECOC::set_ECOC_table (UINT i, const ECOC_VECTOR& p) {
00081 for (UINT c = 0; c < p.size(); ++c) {
00082 if (ecoc[c].size() <= i) {
00083 assert(ecoc[c].size() == i);
00084 ecoc[c].push_back(p[c]);
00085 }
00086 else ecoc[c][i] = p[c];
00087 }
00088 }
00089
00090 void MultiClass_ECOC::set_ECOC_table (ECOC_TYPE et) {
00091 assert(nclass > 1);
00092 ECOC_TABLE e;
00093
00094 switch (et) {
00095 case ONE_VS_ONE: {
00096 UINT m = nclass*(nclass-1)/2;
00097 e = ECOC_TABLE(nclass, ECOC_VECTOR(m, 0));
00098 UINT k = 0;
00099 for (UINT i = 0; i < nclass; ++i)
00100 for (UINT j = i+1; j < nclass; ++j, ++k) {
00101 e[i][k] = 1; e[j][k] = -1;
00102 }
00103 }
00104 break;
00105
00106 case ONE_VS_ALL:
00107 e = ECOC_TABLE(nclass, ECOC_VECTOR(nclass, -1));
00108 for (UINT i = 0; i < nclass; ++i)
00109 e[i][i] = 1;
00110 break;
00111
00112 default:
00113 assert(false);
00114 }
00115
00116 set_ECOC_table(e);
00117 }
00118
00119
00120 REAL MultiClass_ECOC::c_error (const Output& out, const Output& y) const {
00121 assert(n_output() == 1);
00122 return !LABEL_EQUAL(out[0], y[0]);
00123 }
00124
00125 void MultiClass_ECOC::initialize () {
00126 Aggregating::initialize();
00127 lm_wgt.clear();
00128 #if MULTICLASS_ECOC_OUTPUT_CACHE
00129 clear_cache();
00130 #endif
00131 }
00132
00133 void MultiClass_ECOC::set_train_data (const pDataSet& pd,
00134 const pDataWgt& pw) {
00135 assert(!lm_base || n_output() == 1);
00136 Aggregating::set_train_data(pd, pw);
00137
00138 std::set<REAL> ls;
00139 for (UINT i = 0; i < n_samples; ++i)
00140 ls.insert(ptd->y(i)[0]);
00141
00142
00143 std::vector<REAL> new_labels(ls.begin(), ls.end());
00144 if (nclass > 0) {
00145 bool same = (nclass == new_labels.size());
00146 for (UINT c = 0; same && c < nclass; ++c)
00147 same = LABEL_EQUAL(labels[c], new_labels[c]);
00148 if (!same)
00149 std::cerr << "MultiClass_ECOC: Warning: "
00150 "Class label mapping is changed\n";
00151 }
00152
00153
00154 labels.swap(new_labels);
00155 nclass = labels.size();
00156 for (UINT c = 1; c < nclass; ++c)
00157 if (LABEL_EQUAL(labels[c-1], labels[c])) {
00158 std::cerr << "MultiClass_ECOC: Error: Labels ("
00159 << labels[c-1] << " and " << labels[c]
00160 << ") are too close. Alter LABEL_EQUAL?\n";
00161 std::exit(-1);
00162 }
00163
00164
00165 ex_class.resize(n_samples);
00166 for (UINT i = 0; i < n_samples; ++i) {
00167 REAL y = ptd->y(i)[0];
00168 UINT c = LABEL2INDEX(y);
00169 assert(c < nclass && LABEL_EQUAL(y, labels[c]));
00170 ex_class[i] = c;
00171 }
00172
00173 ecoc.resize(nclass);
00174 #if MULTICLASS_ECOC_OUTPUT_CACHE
00175 clear_cache();
00176 #endif
00177 local_d.resize(nclass);
00178 }
00179
00180 REAL MultiClass_ECOC::train () {
00181 assert(ptd != 0 && ptw != 0);
00182 assert(lm_base != 0);
00183 assert(_n_out == 1);
00184 assert(nclass <= ecoc.size() && nclass == labels.size());
00185
00186
00187 n_in_agg = lm.size();
00188 assert(n_in_agg == lm_wgt.size());
00189 reset_training();
00190
00191 while (n_in_agg < max_n_model) {
00192 ECOC_VECTOR par;
00193 if (!ECOC_partition(n_in_agg, par)) break;
00194 #if VERBOSE_OUTPUT
00195 std::cout << "*** " << id() << " #" << n_in_agg+1
00196 << " / " << max_n_model << " ***\n";
00197 #endif
00198
00199 const pLearnModel p = train_with_partition(par);
00200
00201 const REAL w = assign_weight(par, *p);
00202 if (w <= 0) break;
00203
00204 lm.push_back(p); lm_wgt.push_back(w);
00205 set_ECOC_table(n_in_agg, par);
00206 ++n_in_agg;
00207 update_training(par);
00208 }
00209
00210 return 0;
00211 }
00212
00213
00214 #define GET_BEST_CLASS(distance_to_class_c) \
00215 REAL dmin = INFINITY; UINT cmin = UINT(-1); \
00216 for (UINT c = 0; c < nclass; ++c) { \
00217 REAL dc = distance_to_class_c; \
00218 assert(dc < INFINITY/10); \
00219 if (dc < dmin) { dmin = dc; cmin = c; } \
00220 }
00221 #define GET_MARGIN(distance_to_class_c,y) \
00222 REAL dy = 0, dmin = INFINITY; \
00223 for (UINT c = 0; c < nclass; ++c) { \
00224 REAL dc = distance_to_class_c; \
00225 assert(dc < INFINITY/10); \
00226 if (c == y) dy = dc; \
00227 else if (dc < dmin) dmin = dc; \
00228 }
00229
00230 const std::vector<REAL>& MultiClass_ECOC::distances (const Input& x) const {
00231 assert(n_in_agg <= lm.size() && n_in_agg <= lm_wgt.size());
00232 #ifndef NDEBUG
00233 for (UINT i = 0; i < n_in_agg; ++i)
00234 assert(lm_wgt[i] >= 0);
00235 #endif
00236
00237 assert(n_output() == 1);
00238 Output out(n_in_agg);
00239 for (UINT i = 0; i < n_in_agg; ++i) {
00240 assert(lm[i] != 0);
00241 out[i] = (*lm[i])(x)[0];
00242 }
00243 std::vector<REAL>& d = local_d;
00244 assert(d.size() == nclass);
00245 for (UINT c = 0; c < nclass; ++c)
00246 d[c] = ECOC_distance(out, ecoc[c]);
00247 return d;
00248 }
00249
00250 const std::vector<REAL>& MultiClass_ECOC::distances (UINT idx) const {
00251 #if MULTICLASS_ECOC_OUTPUT_CACHE == 2 // distance cache
00252 assert(cache_n.size() == n_samples);
00253 if (cache_n[idx] > n_in_agg)
00254 clear_cache(idx);
00255 std::vector<REAL>& d = cache_d[idx];
00256 assert(d.size() == nclass);
00257 for (UINT i = cache_n[idx]; i < n_in_agg; ++i) {
00258 REAL o = lm[i]->get_output(idx)[0];
00259 for (UINT c = 0; c < nclass; ++c)
00260 d[c] = ECOC_distance(o, ecoc[c][i], lm_wgt[i], d[c]);
00261 }
00262 cache_n[idx] = n_in_agg;
00263 #else // compute the output first
00264 #if MULTICLASS_ECOC_OUTPUT_CACHE == 1
00265 assert(cache_o.size() == n_samples);
00266 Output& out = cache_o[idx];
00267 for (UINT i = out.size(); i < n_in_agg; ++i) {
00268 assert(lm[i] != 0);
00269 out.push_back(lm[i]->get_output(idx)[0]);
00270 }
00271 #elif !MULTICLASS_ECOC_OUTPUT_CACHE
00272 Output out(n_in_agg);
00273 for (UINT i = 0; i < n_in_agg; ++i) {
00274 assert(lm[i] != 0);
00275 out[i] = lm[i]->get_output(idx)[0];
00276 }
00277 #else
00278 #error "Wrong value of MULTICLASS_ECOC_OUTPUT_CACHE"
00279 #endif
00280 std::vector<REAL>& d = local_d;
00281 assert(d.size() == nclass);
00282 for (UINT c = 0; c < nclass; ++c)
00283 d[c] = ECOC_distance(out, ecoc[c]);
00284 #endif
00285 return d;
00286 }
00287
00288 Output MultiClass_ECOC::operator() (const Input& x) const {
00289 const std::vector<REAL>& d = distances(x);
00290 GET_BEST_CLASS(d[c]);
00291 return Output(1, labels[cmin]);
00292 }
00293
00294 Output MultiClass_ECOC::get_output (UINT idx) const {
00295 assert(ptw != NULL);
00296 assert(nclass <= ecoc.size() && nclass == labels.size());
00297
00298 const std::vector<REAL>& d = distances(idx);
00299 GET_BEST_CLASS(d[c]);
00300
00301
00302
00303
00304 return Output(1, labels[cmin]);
00305 }
00306
00307 REAL MultiClass_ECOC::margin_of (const Input& x, const Output& l) const {
00308 UINT y = LABEL2INDEX(l[0]);
00309 assert(y < nclass && LABEL_EQUAL(l[0], labels[y]));
00310 const std::vector<REAL>& d = distances(x);
00311 GET_MARGIN(d[c], y);
00312 return dmin - dy;
00313 }
00314
00315 REAL MultiClass_ECOC::margin (UINT idx) const {
00316 assert(ptw != NULL);
00317 UINT y = ex_class[idx];
00318
00319 const std::vector<REAL>& d = distances(idx);
00320 GET_MARGIN(d[c], y);
00321 return dmin - dy;
00322 }
00323
00324 REAL MultiClass_ECOC::ECOC_distance (const Output& o,
00325 const ECOC_VECTOR& cw) const {
00326 assert(n_in_agg <= o.size() && n_in_agg <= cw.size());
00327 assert(n_in_agg <= lm_wgt.size());
00328 REAL ip = 0;
00329 for (UINT i = 0; i < n_in_agg; ++i)
00330 ip += lm_wgt[i] * (o[i]>0? -.5:.5) * cw[i];
00331 return ip;
00332 }
00333
00334 #if MULTICLASS_ECOC_OUTPUT_CACHE == 2 // distance cache
00335 REAL MultiClass_ECOC::ECOC_distance (REAL o, int cw, REAL w, REAL d) const {
00336 return d + w * (o>0? -.5:.5) * cw;
00337 }
00338 #endif
00339
00340 bool MultiClass_ECOC::ECOC_partition (UINT i, ECOC_VECTOR& p) {
00341 assert(nclass <= ecoc.size());
00342 p.resize(nclass);
00343 for (UINT c = 0; c < nclass; ++c) {
00344 if (i >= ecoc[c].size()) return false;
00345 p[c] = ecoc[c][i];
00346 }
00347 return true;
00348 }
00349
00350 pLearnModel MultiClass_ECOC::train_with_partition (ECOC_VECTOR& p) {
00351 LearnModel *plm = lm_base->clone();
00352 assert(plm != 0);
00353
00354
00355 DataSet* btd = new DataSet();
00356 DataWgt* btw = new DataWgt();
00357 REAL wsum = 0;
00358 for (UINT i = 0; i < n_samples; ++i) {
00359 const Input& x = ptd->x(i);
00360 int b = p[ex_class[i]];
00361 if (b != 0) {
00362 assert(b == 1 || b == -1);
00363 btd->append(x, Output(1, b));
00364 btw->push_back((*ptw)[i]);
00365 wsum += (*ptw)[i];
00366 }
00367 }
00368 assert(wsum > 0);
00369 for (UINT i = 0; i < btw->size(); ++i)
00370 (*btw)[i] /= wsum;
00371
00372 plm->set_train_data(btd, btw);
00373 plm->train();
00374
00375 plm->set_train_data(ptd, ptw);
00376 return plm;
00377 }
00378
00379 REAL MultiClass_ECOC::assign_weight (const ECOC_VECTOR&, const LearnModel&) {
00380 return 1;
00381 }
00382
00383 REAL MultiClass_ECOC::cost () const {
00384 assert(ptd != 0 && ptw != 0);
00385 REAL cst = 0;
00386 for (UINT i = 0; i < n_samples; ++i) {
00387 const std::vector<REAL>& d = distances(i);
00388 const UINT y = ex_class[i];
00389 REAL dy = d[y], csti = 0;
00390 for (UINT c = 0; c < nclass; ++c)
00391 if (c != y) {
00392 csti += std::exp(dy - d[c]);
00393 }
00394 cst += (*ptw)[i] * csti;
00395 }
00396 return cst;
00397 }
00398
00399 }