Compute metrics by site

Imports

import seaborn as sns
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, balanced_accuracy_score, f1_score, roc_auc_score
from sklearn.model_selection import train_test_split

from uniharmony import verbosity
from uniharmony.datasets import make_multisite_classification
from uniharmony.metrics import report_metrics_by_site


sns.set_theme(style="whitegrid")
verbosity("error")

clf = LogisticRegression()

Data generation

X, y, sites = make_multisite_classification(n_sites=10, signal_strength=0.5)

X_train, X_test, y_train, y_test, sites_train, sites_test = train_test_split(X, y, sites)

Metrics by site report

clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
y_scores = clf.predict_proba(X_test)[:, 0]
metrics = report_metrics_by_site(y_test, y_pred, sites_test, balanced_accuracy_score)

# for key in metrics.keys():
#     print(f"For site {key}: bACC {metrics[key]:.4}")

# Compute metrics but now request the overall
metrics = report_metrics_by_site(y_test, y_pred, sites_test, balanced_accuracy_score, overall_performance=True)

# Compute the metric outside the function to compare.
bacc = balanced_accuracy_score(y_true=y_test, y_pred=y_pred)

# Overall comparison.
print(f"Overall bACC: {bacc}")
# The overall performance is also stored in the metrics if requested.
print(f"Overall bACC: {metrics['balanced_accuracy_score']['overall']}")

Overall bACC: 0.6154589371980677
Overall bACC: 0.6154589371980677

If requested, the function also computes the overall performance and stores it as another entry in the dictionary.

# Single metric (simplest case)
metrics = report_metrics_by_site(y_test, y_scores, sites_test, accuracy_score)
print(metrics)

# Single metric with kwargs
metrics = report_metrics_by_site(y_test, y_scores, sites_test, f1_score, metric_kwargs={"threshold": 0.5, "average": "macro"})
print(metrics)

# Multiple metrics
metrics = report_metrics_by_site(
    y_test,
    y_scores,
    sites_test,
    metrics=[roc_auc_score, accuracy_score],
    metric_kwargs=[{}, {"threshold": 0.5}],
)
print(metrics)

# With overall performance
metrics = report_metrics_by_site(
    y_test,
    y_scores,
    sites_test,
    metrics=[accuracy_score, roc_auc_score],
    overall_performance=True,
)
print(metrics)

{'accuracy_score': {'overall': 0.384, 0: 0.3103448275862069, 1: 0.4444444444444444, 2: 0.36666666666666664, 3: 0.05, 4: 0.37037037037037035, 5: 0.6333333333333333, 6: 0.44, 7: 0.2222222222222222, 8: 0.5909090909090909, 9: 0.36363636363636365}}
{'f1_score': {'overall': 0.38364497950819676, 0: 0.3028846153846154, 1: 0.375, 2: 0.3659621802002225, 3: 0.047619047619047616, 4: 0.36689655172413793, 5: 0.6296296296296295, 6: 0.3923611111111111, 7: 0.20476858345021037, 8: 0.5686274509803921, 9: 0.3418803418803419}}
{'roc_auc_score': {'overall': 0.3285024154589372, 0: 0.3317307692307692, 1: 0.3625, 2: 0.38009049773755654, 3: 0.11111111111111116, 4: 0.29670329670329676, 5: 0.4955357142857143, 6: 0.16883116883116883, 7: 0.25555555555555554, 8: 0.46153846153846156, 9: 0.18181818181818177}, 'accuracy_score': {'overall': 0.384, 0: 0.3103448275862069, 1: 0.4444444444444444, 2: 0.36666666666666664, 3: 0.05, 4: 0.37037037037037035, 5: 0.6333333333333333, 6: 0.44, 7: 0.2222222222222222, 8: 0.5909090909090909, 9: 0.36363636363636365}}
{'accuracy_score': {'overall': 0.384, 0: 0.3103448275862069, 1: 0.4444444444444444, 2: 0.36666666666666664, 3: 0.05, 4: 0.37037037037037035, 5: 0.6333333333333333, 6: 0.44, 7: 0.2222222222222222, 8: 0.5909090909090909, 9: 0.36363636363636365}, 'roc_auc_score': {'overall': 0.3285024154589372, 0: 0.3317307692307692, 1: 0.3625, 2: 0.38009049773755654, 3: 0.11111111111111116, 4: 0.29670329670329676, 5: 0.4955357142857143, 6: 0.16883116883116883, 7: 0.25555555555555554, 8: 0.46153846153846156, 9: 0.18181818181818177}}