From 20bd47eea4cd792b6407fb1819ef4ba004841d21 Mon Sep 17 00:00:00 2001 From: Jan Sosulski Date: Thu, 6 Feb 2020 16:45:18 +0100 Subject: [PATCH 1/3] errors in validation are now handled via error_score parameter --- moabb/evaluations/evaluations.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/moabb/evaluations/evaluations.py b/moabb/evaluations/evaluations.py index a381da093..150b53f6c 100644 --- a/moabb/evaluations/evaluations.py +++ b/moabb/evaluations/evaluations.py @@ -61,9 +61,15 @@ def score(self, clf, X, y, scoring): le = LabelEncoder() y = le.fit_transform(y) - acc = cross_val_score(clf, X, y, cv=cv, scoring=scoring, - n_jobs=self.n_jobs, error_score=self.error_score) - return acc.mean() + try: + acc = cross_val_score(clf, X, y, cv=cv, scoring=scoring, + n_jobs=self.n_jobs, error_score=self.error_score).mean() + except ValueError as e: + if self.error_score == 'raise': + raise e + elif self.error_score is np.nan: + acc = np.nan + return acc def is_valid(self, dataset): return True From 5d1c3ac8bd430586d5aa4bbbb726a0c661d0cf68 Mon Sep 17 00:00:00 2001 From: Jan Sosulski Date: Thu, 6 Feb 2020 16:45:18 +0100 Subject: [PATCH 2/3] errors in validation are now handled via error_score parameter --- moabb/evaluations/evaluations.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/moabb/evaluations/evaluations.py b/moabb/evaluations/evaluations.py index a381da093..150b53f6c 100644 --- a/moabb/evaluations/evaluations.py +++ b/moabb/evaluations/evaluations.py @@ -61,9 +61,15 @@ def score(self, clf, X, y, scoring): le = LabelEncoder() y = le.fit_transform(y) - acc = cross_val_score(clf, X, y, cv=cv, scoring=scoring, - n_jobs=self.n_jobs, error_score=self.error_score) - return acc.mean() + try: + acc = cross_val_score(clf, X, y, cv=cv, scoring=scoring, + n_jobs=self.n_jobs, error_score=self.error_score).mean() + except ValueError as e: + if self.error_score == 'raise': + raise e + elif self.error_score is np.nan: + acc = np.nan + return acc def is_valid(self, dataset): return True From 0f534f9cf675bbc51e92d0c3a93abacb91d26de9 Mon Sep 17 00:00:00 2001 From: Jan Sosulski Date: Thu, 4 Jun 2020 12:21:34 +0200 Subject: [PATCH 3/3] add error_score to other two evaluations --- moabb/evaluations/evaluations.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/moabb/evaluations/evaluations.py b/moabb/evaluations/evaluations.py index 150b53f6c..fdf112ea6 100644 --- a/moabb/evaluations/evaluations.py +++ b/moabb/evaluations/evaluations.py @@ -109,11 +109,17 @@ def evaluate(self, dataset, pipelines): cv = LeaveOneGroupOut() for train, test in cv.split(X, y, groups): t_start = time() - score = _fit_and_score(clone(clf), X, y, scorer, train, - test, verbose=False, - parameters=None, - fit_params=None, - error_score=self.error_score)[0] + try: + score = _fit_and_score(clone(clf), X, y, scorer, train, + test, verbose=False, + parameters=None, + fit_params=None, + error_score=self.error_score)[0] + except ValueError as e: + if self.error_score == 'raise': + raise e + elif self.error_score is np.nan: + score = np.nan duration = time() - t_start res = {'time': duration, 'dataset': dataset, @@ -182,7 +188,13 @@ def evaluate(self, dataset, pipelines): # we eval on each session for session in np.unique(sessions[test]): ix = sessions[test] == session - score = _score(model, X[test[ix]], y[test[ix]], scorer) + try: + score = _score(model, X[test[ix]], y[test[ix]], scorer) + except ValueError as e: + if self.error_score == 'raise': + raise e + elif self.error_score is np.nan: + score = np.nan res = {'time': duration, 'dataset': dataset,