Skip to content

Commit

Permalink
updated KDEyML-Loss to fix NaN issues and updated experiment
Browse files Browse the repository at this point in the history
  • Loading branch information
@tobiaslotz
tobiaslotz committed Mar 6, 2024
1 parent 416111c commit 0dbadce
Show file tree
Hide file tree
Showing 2 changed files with 152 additions and 145 deletions.
294 changes: 150 additions & 144 deletions qunfold/experiments/lequa.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ def trial(trial_config, trn_data, val_gen, tst_gen, seed, n_trials):
protocol = val_gen,
error = "m" + error_metric, # ae -> mae, rae -> mrae
refit = False,
# verbose = True,
verbose = True,
).fit(trn_data)
parameters = quapy_method.best_params_
val_error = quapy_method.best_score_
Expand Down Expand Up @@ -138,101 +138,101 @@ def main(
"classifier__C": clf_grid["transformer__classifier__estimator__C"],
}
methods = [ # (method_name, package, method, param_grid)
("ACC", "qunfold", QuaPyWrapper(ACC(clf, seed=seed)), clf_grid),
("ACC", "QuaPy", qp.method.aggregative.ACC(qp_clf, val_split=5), qp_clf_grid),
("PACC", "qunfold", QuaPyWrapper(PACC(clf, seed=seed)), clf_grid),
("PACC", "QuaPy", qp.method.aggregative.PACC(qp_clf, val_split=5), qp_clf_grid),
("HDy", "qunfold",
QuaPyWrapper(HDy(clf, 2, seed=seed)),
{
"transformer__preprocessor__classifier__estimator__C":
clf_grid["transformer__classifier__estimator__C"],
"transformer__n_bins": [2, 4, 6],
}
),
("HDy", "QuaPy",
qp.method.aggregative.DistributionMatching(
classifier = qp_clf,
divergence = 'HD',
cdf = False
),
dict(qp_clf_grid, nbins = [2, 4, 6]) # extend the qp_clf_grid
),
("EDy", "qunfold",
QuaPyWrapper(EDy(clf, seed=seed)),
{
"transformer__preprocessor__classifier__estimator__C":
clf_grid["transformer__classifier__estimator__C"],
}
),
("RUN", "qunfold", QuaPyWrapper(RUN(ClassTransformer(clf), seed=seed)), clf_grid),
("KMMe", "qunfold", QuaPyWrapper(KMM(kernel="energy", seed=seed)), None),
("KMMey", "qunfold", # KMM with the energy kernel after classification
QuaPyWrapper(GenericMethod(
LeastSquaresLoss(),
EnergyKernelTransformer(preprocessor=ClassTransformer(
clf,
is_probabilistic = True,
)),
seed = seed
)),
{
"transformer__preprocessor__classifier__estimator__C":
clf_grid["transformer__classifier__estimator__C"],
}
),
("KMMr", "qunfold",
QuaPyWrapper(KMM(kernel="rff", seed=seed)),
{ "transformer__sigma": [1e-2, 1e-1, 1e0, 1e1, 1e2] }
),
("KMMry", "qunfold", # KMM with the Gaussian RFF kernel after classification
QuaPyWrapper(GenericMethod(
LeastSquaresLoss(),
GaussianRFFKernelTransformer(
seed = seed,
preprocessor = ClassTransformer(
clf,
is_probabilistic = True,
),
),
seed = seed
)),
{
"transformer__sigma": [1e-2, 1e-1, 1e0, 1e1, 1e2],
"transformer__preprocessor__classifier__estimator__C":
clf_grid["transformer__classifier__estimator__C"],
}
),
#("ACC", "qunfold", QuaPyWrapper(ACC(clf, seed=seed)), clf_grid),
#("ACC", "QuaPy", qp.method.aggregative.ACC(qp_clf, val_split=5), qp_clf_grid),
#("PACC", "qunfold", QuaPyWrapper(PACC(clf, seed=seed)), clf_grid),
#("PACC", "QuaPy", qp.method.aggregative.PACC(qp_clf, val_split=5), qp_clf_grid),
#("HDy", "qunfold",
# QuaPyWrapper(HDy(clf, 2, seed=seed)),
# {
# "transformer__preprocessor__classifier__estimator__C":
# clf_grid["transformer__classifier__estimator__C"],
# "transformer__n_bins": [2, 4, 6],
# }
#),
#("HDy", "QuaPy",
# qp.method.aggregative.DistributionMatching(
# classifier = qp_clf,
# divergence = 'HD',
# cdf = False
# ),
# dict(qp_clf_grid, nbins = [2, 4, 6]) # extend the qp_clf_grid
#),
#("EDy", "qunfold",
# QuaPyWrapper(EDy(clf, seed=seed)),
# {
# "transformer__preprocessor__classifier__estimator__C":
# clf_grid["transformer__classifier__estimator__C"],
# }
#),
#("RUN", "qunfold", QuaPyWrapper(RUN(ClassTransformer(clf), seed=seed)), clf_grid),
#("KMMe", "qunfold", QuaPyWrapper(KMM(kernel="energy", seed=seed)), None),
#("KMMey", "qunfold", # KMM with the energy kernel after classification
# QuaPyWrapper(GenericMethod(
# LeastSquaresLoss(),
# EnergyKernelTransformer(preprocessor=ClassTransformer(
# clf,
# is_probabilistic = True,
# )),
# seed = seed
# )),
# {
# "transformer__preprocessor__classifier__estimator__C":
# clf_grid["transformer__classifier__estimator__C"],
# }
#),
#("KMMr", "qunfold",
# QuaPyWrapper(KMM(kernel="rff", seed=seed)),
# { "transformer__sigma": [1e-2, 1e-1, 1e0, 1e1, 1e2] }
#),
#("KMMry", "qunfold", # KMM with the Gaussian RFF kernel after classification
# QuaPyWrapper(GenericMethod(
# LeastSquaresLoss(),
# GaussianRFFKernelTransformer(
# seed = seed,
# preprocessor = ClassTransformer(
# clf,
# is_probabilistic = True,
# ),
# ),
# seed = seed
# )),
# {
# "transformer__sigma": [1e-2, 1e-1, 1e0, 1e1, 1e2],
# "transformer__preprocessor__classifier__estimator__C":
# clf_grid["transformer__classifier__estimator__C"],
# }
#),
("KDEyML", "QuaPy", qp.method.aggregative.KDEyML(qp_clf, val_split=5),
{
"bandwidth" : [1e-2, 1e-1, 1e0, 1e1, 1e2],
"classifier__C" : [1e-3, 1e-2, 1e-1, 1e0, 1e1]
}
),
("KDEyHD", "QuaPy", qp.method.aggregative.KDEyHD(qp_clf, val_split=5),
{
"bandwidth" : [1e-2, 1e-1, 1e0, 1e1, 1e2],
"classifier__C" : [1e-3, 1e-2, 1e-1, 1e0, 1e1]
}
),
("KDEyCS", "QuaPy", qp.method.aggregative.KDEyCS(qp_clf, val_split=5),
{
"bandwidth" : [1e-2, 1e-1, 1e0, 1e1, 1e2],
"classifier__C" : [1e-3, 1e-2, 1e-1, 1e0, 1e1]
}
),
#("KDEyHD", "qunfold", QuaPyWrapper(KDEyHD(clf, bandwidth=0.1, random_state=seed)),
#("KDEyHD", "QuaPy", qp.method.aggregative.KDEyHD(qp_clf, val_split=5),
# {
# "transformer__bandwidth" : [1e-2, 1e-1, 1e0, 1e1, 1e2],
# **clf_grid,
# "bandwidth" : [1e-2, 1e-1, 1e0, 1e1, 1e2],
# "classifier__C" : [1e-3, 1e-2, 1e-1, 1e0, 1e1]
# }
#),
#("KDEyCS", "QuaPy", qp.method.aggregative.KDEyCS(qp_clf, val_split=5),
# {
# "bandwidth" : [1e-2, 1e-1, 1e0, 1e1, 1e2],
# "classifier__C" : [1e-3, 1e-2, 1e-1, 1e0, 1e1]
# }
#),
#("KDEyML", "qunfold", QuaPyWrapper(KDEyML(clf, bandwidth=0.1)),
#("KDEyHD", "qunfold", QuaPyWrapper(KDEyHD(clf, bandwidth=0.1, random_state=seed)),
# {
# "transformer__bandwidth" : [1e-2, 1e-1, 1e0, 1e1, 1e2],
# **clf_grid,
# }
#),
("KDEyML", "qunfold", QuaPyWrapper(KDEyML(clf, bandwidth=0.1)),
{
"transformer__bandwidth" : [1e-2, 1e-1, 1e0, 1e1, 1e2],
**clf_grid,
}
),
#("KDEyCS", "qunfold", QuaPyWrapper(KDEyCS(clf, bandwidth=0.1)),
# {
# "transformer__bandwidth" : [1e-2, 1e-1, 1e0, 1e1, 1e2],
Expand All @@ -254,76 +254,82 @@ def main(
"classifier__C": clf_grid["transformer__classifier__estimator__C"],
}
methods = [ # (method_name, package, method, param_grid)
("ACC", "qunfold", QuaPyWrapper(ACC(clf, seed=seed)), clf_grid),
("ACC", "QuaPy", qp.method.aggregative.ACC(qp_clf, val_split=3), qp_clf_grid),
("PACC", "qunfold", QuaPyWrapper(PACC(clf, seed=seed)), clf_grid),
("PACC", "QuaPy", qp.method.aggregative.PACC(qp_clf, val_split=3), qp_clf_grid),
("HDy", "qunfold", QuaPyWrapper(HDy(clf, 2, seed=seed)), None),
("HDy", "QuaPy",
qp.method.aggregative.DistributionMatching(
classifier = qp_clf,
divergence = 'HD',
cdf = False
),
None
),
("EDy", "qunfold", QuaPyWrapper(EDy(clf, seed=seed)), None),
("RUN", "qunfold", QuaPyWrapper(RUN(ClassTransformer(clf), seed=seed)), None),
("KMMe", "qunfold", QuaPyWrapper(KMM(kernel="energy", seed=seed)), None),
("KMMey", "qunfold", # KMM with the energy kernel after classification
QuaPyWrapper(GenericMethod(
LeastSquaresLoss(),
EnergyKernelTransformer(preprocessor=ClassTransformer(
clf,
is_probabilistic = True,
)),
seed = seed
)),
{
"transformer__preprocessor__classifier__estimator__C":
clf_grid["transformer__classifier__estimator__C"],
}
),
("KMMr", "qunfold",
QuaPyWrapper(KMM(kernel="rff", seed=seed)),
{ "transformer__sigma": [ 1e-1 ] }
),
("KMMry", "qunfold", # KMM with the Gaussian RFF kernel after classification
QuaPyWrapper(GenericMethod(
LeastSquaresLoss(),
GaussianRFFKernelTransformer(
seed = seed,
preprocessor = ClassTransformer(
clf,
is_probabilistic = True,
),
),
seed = seed
)),
{
"transformer__sigma": [ 1e-1 ],
"transformer__preprocessor__classifier__estimator__C":
clf_grid["transformer__classifier__estimator__C"],
}
),
("KDEyHD", "qunfold", QuaPyWrapper(KDEyHD(clf, bandwidth=0.1, random_state=seed)),
{
"transformer__bandwidth" : [1e-1],
**clf_grid,
}
),
#("ACC", "qunfold", QuaPyWrapper(ACC(clf, seed=seed)), clf_grid),
#("ACC", "QuaPy", qp.method.aggregative.ACC(qp_clf, val_split=3), qp_clf_grid),
#("PACC", "qunfold", QuaPyWrapper(PACC(clf, seed=seed)), clf_grid),
#("PACC", "QuaPy", qp.method.aggregative.PACC(qp_clf, val_split=3), qp_clf_grid),
#("HDy", "qunfold", QuaPyWrapper(HDy(clf, 2, seed=seed)), None),
#("HDy", "QuaPy",
# qp.method.aggregative.DistributionMatching(
# classifier = qp_clf,
# divergence = 'HD',
# cdf = False
# ),
# None
#),
#("EDy", "qunfold", QuaPyWrapper(EDy(clf, seed=seed)), None),
#("RUN", "qunfold", QuaPyWrapper(RUN(ClassTransformer(clf), seed=seed)), None),
#("KMMe", "qunfold", QuaPyWrapper(KMM(kernel="energy", seed=seed)), None),
#("KMMey", "qunfold", # KMM with the energy kernel after classification
# QuaPyWrapper(GenericMethod(
# LeastSquaresLoss(),
# EnergyKernelTransformer(preprocessor=ClassTransformer(
# clf,
# is_probabilistic = True,
# )),
# seed = seed
# )),
# {
# "transformer__preprocessor__classifier__estimator__C":
# clf_grid["transformer__classifier__estimator__C"],
# }
#),
#("KMMr", "qunfold",
# QuaPyWrapper(KMM(kernel="rff", seed=seed)),
# { "transformer__sigma": [ 1e-1 ] }
#),
#("KMMry", "qunfold", # KMM with the Gaussian RFF kernel after classification
# QuaPyWrapper(GenericMethod(
# LeastSquaresLoss(),
# GaussianRFFKernelTransformer(
# seed = seed,
# preprocessor = ClassTransformer(
# clf,
# is_probabilistic = True,
# ),
# ),
# seed = seed
# )),
# {
# "transformer__sigma": [ 1e-1 ],
# "transformer__preprocessor__classifier__estimator__C":
# clf_grid["transformer__classifier__estimator__C"],
# }
#),
#("KDEyHD", "qunfold", QuaPyWrapper(KDEyHD(clf, bandwidth=0.1, random_state=seed)),
#{
# "transformer__bandwidth" : [1e-1],
# **clf_grid,
#}
#),
("KDEyML", "qunfold", QuaPyWrapper(KDEyML(clf, bandwidth=0.1)),
{
"transformer__bandwidth" : [1e-1],
"transformer__bandwidth" : [1e-1, 1e-2],
**clf_grid,
}
),
("KDEyCS", "qunfold", QuaPyWrapper(KDEyCS(clf, bandwidth=0.1)),
("KDEyML", "QuaPy", qp.method.aggregative.KDEyML(qp_clf, val_split=5),
{
"transformer__bandwidth" : [1e-1],
**clf_grid,
"bandwidth" : [1e-2, 1e-1],
"classifier__C" : [1e1]
}
),
#("KDEyCS", "qunfold", QuaPyWrapper(KDEyCS(clf, bandwidth=0.1)),
# {
# "transformer__bandwidth" : [1e-1],
# **clf_grid,
# }
#),
]
trn_data = trn_data.split_stratified(3000, random_state=seed)[0] # subsample
val_gen.true_prevs.df = val_gen.true_prevs.df[:3] # use only 3 validation samples
Expand Down
3 changes: 2 additions & 1 deletion qunfold/losses.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ def _kde_hd_loss(p, q, M, N=None):
# (negative log-likelihood function)
# Parameter M is redundant
def _kde_ml_loss(p, q, M, N=None):
return -jnp.sum(jnp.log(jnp.dot(p, q) + 1e-10))
i = jnp.logical_and(jnp.any(q > 0, axis=0), jnp.all(jnp.isfinite(q), axis=0))
return -jnp.sum(jnp.log(jnp.dot(p, q[:, i]) + 1e-10))

# helper function for Boolean masks M[_nonzero_features(M),:] and q[_nonzero_features(M)]
def _nonzero_features(M):
Expand Down

0 comments on commit 0dbadce

Please sign in to comment.