tools.py

from sklearn import tree
import matplotlib.pyplot as plt
import numpy as np
from sklearn import metrics


def plot_tree(dt_clf, attributes):
    plt.figure(figsize=(25, 20))
    _ = tree.plot_tree(dt_clf,
                       feature_names=attributes[:-1, 0],
                       class_names=attributes[-1][1],
                       filled=True)
    plt.show()


def rmse_loss(P, Y):
    """
    Root mean squared error
    :param P: probabilities, array-like of shape (n_samples, n_classes)
    :param Y: labels in one-hot format, array-like of shape (n_samples, n_classes)
    :return:
    """
    return np.average(np.square(P-Y)) ** 0.5


def auc(P, Y):
    """
    Area under curve
    :param P: probabilities, array-like of shape (n_samples, n_classes)
    :param Y: labels in one-hot format, array-like of shape (n_samples, n_classes)
    :return:
    """
    auc_sum = 0
    weight_sum = 0
    for class_i in range(Y.shape[1]):
        P_i = P[:, class_i]
        Y_i = Y[:, class_i]
        w = np.where(Y_i == 1)[0].shape[0]
        if w == 0:
            continue
        fpr, tpr, _ = metrics.roc_curve(Y_i, P_i)
        auc_sum += metrics.auc(fpr, tpr) * w
        weight_sum += w
    return auc_sum / weight_sum


def classification_loss(P, Y):
    """
    Classification loss using the maximum probability
    :param P: probabilities, array-like of shape (n_samples, n_classes)
    :param Y: labels in one-hot format, array-like of shape (n_samples, n_classes)
    :return:
    """
    loss = 0
    for i in range(len(P)):
        if np.argmax(P[i]) != np.argmax(Y[i]):
            loss += 1

    return loss / len(P)


def pce_avg(partitions, norm=1):
    """
    Partition calibration error (PCE) for binary classification

    :param partitions: 2d list of (p, y) pairs each of which represents the predicted probability and true label (0 or 1)
    each row of partitions represents the samples of one bin
    :param norm: p in lp-norm for PCE

    :return: weighted average of PCE of bins
    """

    assert norm >= 1 and norm % 1 == 0

    weight_sum = 0
    weighted_pce_sum = 0.0

    for partition in partitions:
        prob_sum = 0
        label_sum = 0  # = number of positive samples
        for s in partition:
            prob_sum += s[0]
            label_sum += s[1]

        pce = abs(prob_sum - label_sum) / len(partition)

        weighted_pce_sum += (pce ** norm) * len(partition)
        weight_sum += len(partition)

    return (weighted_pce_sum / weight_sum) ** (1/norm)


def ppd_avg(partitions, norm=1):
    """
    Partition probability deviation (PPD) for binary classification
     PPD measures the average deviation of each score in a bin from their average label

    :param partitions: 2d list of (p, y) pairs each of which represents the predicted probability and true label (0 or 1)
    each row of partitions represents the samples of one bin
    :param norm: p in lp-norm for PPD

    :return: weighted average of PPD
    """

    assert norm >= 1 and norm % 1 == 0

    weight_sum = 0
    weighted_ppd_sum = 0.0

    for partition in partitions:
        label_sum = 0  # = number of positive samples
        for s in partition:
            label_sum += s[1]

        label_avg = label_sum / len(partition)

        deviation_sum = 0
        for s in partition:
            prob = s[0]
            deviation_sum += abs(prob - label_avg)

        ppd = deviation_sum / len(partition)

        weighted_ppd_sum += (ppd ** norm) * len(partition)
        weight_sum += len(partition)

    return (weighted_ppd_sum / weight_sum) ** (1 / norm)


def classwise_ece(P, Y, n_bins, norm=1):
    """
    Classwise expected calibration error

    :param P: probabilities, array-like of shape (n_samples, n_classes)
    :param Y: labels in one-hot format, array-like of shape (n_samples, n_classes)
    :param n_bins: number of bins samples are being partitioned to
    :param norm: p in lp-norm for PCE
    :return:
    """
    ece_sum = 0
    n_classes = Y.shape[1]
    for class_i in range(n_classes):
        P_i = P[:, class_i]
        Y_i = Y[:, class_i]
        pairs = np.stack((P_i, Y_i), axis=1)
        sorted_pairs = np.sort(pairs, axis=0)  # sort along the probs
        partitions = np.array_split(sorted_pairs, n_bins)
        ece_sum += pce_avg(partitions, norm)
    return ece_sum / n_classes


def classwise_ece_fixed_bins(P, Y, bins, norm=1):
    """
    Classwise expected calibration error with prefixed bins

    :param P: probabilities, array-like of shape (n_samples, n_classes)
    :param Y: labels in one-hot format, array-like of shape (n_samples, n_classes)
    :param bins: mapping from bins to sample_ids, 2d array [b1:[sample_id, ...], b2, ...]
    :param norm: p in lp-norm for PCE
    :return: ECE
    """

    ece_sum = 0
    n_classes = Y.shape[1]
    for class_i in range(n_classes):
        P_i = P[:, class_i]
        Y_i = Y[:, class_i]
        pairs = np.stack((P_i, Y_i), axis=1)
        partitions = []
        for bin in bins:
            partition = []
            for s_i in bin:
                partition.append(pairs[s_i])
            partitions.append(partition)

        ece_sum += pce_avg(partitions, norm)
    return ece_sum / n_classes


def classwise_pde(P, Y, bins, norm=1):
    """
    Classwise probability deviation error (with prefixed bins)
     It measures the average deviation of each probability in a bin from their average label for each specific class.
     Then an average is taken over the average deviation in all classes.

    :param P: probabilities, array-like of shape (n_samples, n_classes)
    :param Y: labels in one-hot format, array-like of shape (n_samples, n_classes)
    :param bins: mapping from bins to sample_ids, 2d array [b1:[sample_id, ...], b2, ...]
    :param norm: p in lp-norm for PPD
    :return: PDE
    """

    pde_sum = 0
    n_classes = Y.shape[1]
    for class_i in range(n_classes):
        P_i = P[:, class_i]
        Y_i = Y[:, class_i]
        pairs = np.stack((P_i, Y_i), axis=1)
        partitions = []
        for bin in bins:
            partition = []
            for s_i in bin:
                partition.append(pairs[s_i])
            partitions.append(partition)

        pde_sum += ppd_avg(partitions, norm)
    return pde_sum / n_classes


def bins_number_to_bins_partition(bins_number):
    """

    :param bins_number: mapping from sample id to bin id, array-like of shape (n_samples,)
    :return bins: mapping from bins to sample ids
    """
    bin_to_sample = dict()
    for s_i in range(len(bins_number)):
        b = bins_number[s_i]
        if b in bin_to_sample:
            bin_to_sample[b].append(s_i)
        else:
            bin_to_sample[b] = [s_i]

    return list(bin_to_sample.values())


def scores_to_bins(P):
    """
    create bins such that the points with exact same probabilities for all classes are in the same bin
    :param P: :param P: probabilities, array-like of shape (n_samples, n_classes)
    :return: bins: mapping from bins to sample id
    """

    bins = []  # 2d array of sample ids
    values = []  # 1d array of the prob assigned to a bin
    for sample_id in range(len(P)):
        added = False
        for b_i, v in enumerate(values):
            if all(v == P[sample_id]):
                bins[b_i].append(sample_id)
                added = True
                break
        if not added:
            bins.append([sample_id])
            values.append(P[sample_id])

    return bins


def labels_to_one_hot(y, classes):
    """
    :param y: labels, array-like of shape (n_samples,)
    :param classes: list of classes
    :return: Y: one-hot vectors representing labels, array-like of shape (n_samples, n_classes)
    """
    Y = np.zeros((len(y), len(classes)), dtype=int)
    Y[np.where([y[i] == classes for i in range(len(y))])] = 1
    return Y