Skip to content

confusion_matrix

ConfusionMatrix

Bases: Trace

Computes the confusion matrix between y_true (rows) and y_predicted (columns).

Parameters:

Name Type Description Default
true_key str

Name of the key that corresponds to ground truth in the batch dictionary.

 required
pred_key str

Name of the key that corresponds to predicted score in the batch dictionary.

 required
num_classes int

Total number of classes of the confusion matrix.

 required
mode Union[None, str, Iterable[str]]

What mode(s) to execute this Trace in. For example, "train", "eval", "test", or "infer". To execute regardless of mode, pass None. To execute in all modes except for a particular one, you can pass an argument like "!infer" or "!train".

 ('eval', 'test')
ds_id Union[None, str, Iterable[str]]

What dataset id(s) to execute this Trace in. To execute regardless of ds_id, pass None. To execute in all ds_ids except for a particular one, you can pass an argument like "!ds1".

 None
output_name str

Name of the key to store to the state.

 'confusion_matrix'
per_ds bool

Whether to automatically compute this metric individually for every ds_id it runs on, in addition to computing an aggregate across all ds_ids on which it runs. This is automatically False if output_name contains a "|" character.

 True
**kwargs

Additional keyword arguments that pass to sklearn.metrics.confusion_matrix()

 {}

Raises:

Type Description
ValueError

One of ["y_pred", "y_true", "labels"] argument exists in kwargs.
Source code in fastestimator/fastestimator/trace/metric/confusion_matrix.py 
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
@per_ds
@traceable()
class ConfusionMatrix(Trace):
    """Computes the confusion matrix between y_true (rows) and y_predicted (columns).

    Args:
        true_key: Name of the key that corresponds to ground truth in the batch dictionary.
        pred_key: Name of the key that corresponds to predicted score in the batch dictionary.
        num_classes: Total number of classes of the confusion matrix.
        mode: What mode(s) to execute this Trace in. For example, "train", "eval", "test", or "infer". To execute
            regardless of mode, pass None. To execute in all modes except for a particular one, you can pass an argument
            like "!infer" or "!train".
        ds_id: What dataset id(s) to execute this Trace in. To execute regardless of ds_id, pass None. To execute in all
            ds_ids except for a particular one, you can pass an argument like "!ds1".
        output_name: Name of the key to store to the state.
        per_ds: Whether to automatically compute this metric individually for every ds_id it runs on, in addition to
            computing an aggregate across all ds_ids on which it runs. This is automatically False if `output_name`
            contains a "|" character.
        **kwargs: Additional keyword arguments that pass to sklearn.metrics.confusion_matrix()

    Raises:
        ValueError: One of ["y_pred", "y_true", "labels"] argument exists in `kwargs`.
    """
    def __init__(self,
                 true_key: str,
                 pred_key: str,
                 num_classes: int,
                 mode: Union[None, str, Iterable[str]] = ("eval", "test"),
                 ds_id: Union[None, str, Iterable[str]] = None,
                 output_name: str = "confusion_matrix",
                 per_ds: bool = True,
                 **kwargs) -> None:
        ConfusionMatrix.check_kwargs(kwargs)
        super().__init__(inputs=(true_key, pred_key), outputs=output_name, mode=mode, ds_id=ds_id)
        self.num_classes = num_classes
        self.matrix = None
        self.kwargs = kwargs
        self.per_ds = per_ds

    @property
    def true_key(self) -> str:
        return self.inputs[0]

    @property
    def pred_key(self) -> str:
        return self.inputs[1]

    def on_epoch_begin(self, data: Data) -> None:
        self.matrix = None

    def on_batch_end(self, data: Data) -> None:
        y_true, y_pred = to_number(data[self.true_key]), to_number(data[self.pred_key])
        if y_true.shape[-1] > 1 and y_true.ndim > 1:
            y_true = np.argmax(y_true, axis=-1)
        if y_pred.shape[-1] > 1 and y_pred.ndim > 1:
            y_pred = np.argmax(y_pred, axis=-1)
        else:
            y_pred = np.round(y_pred)
        assert y_pred.size == y_true.size

        batch_confusion = confusion_matrix(y_true, y_pred, labels=list(range(0, self.num_classes)), **self.kwargs)

        if self.matrix is None:
            self.matrix = batch_confusion
        else:
            self.matrix += batch_confusion

    def on_epoch_end(self, data: Data) -> None:
        data.write_with_log(self.outputs[0], self.matrix)

    @staticmethod
    def check_kwargs(kwargs: Dict[str, Any]) -> None:
        """Check if `kwargs` has any blacklist argument and raise an error if it does.

        Args:
            kwargs: Keywork arguments to be examined.

        Raises:
            ValueError: One of ["y_pred", "y_true", "labels"] argument exists in `kwargs`.
        """
        blacklist = ["y_true", "y_pred", "labels"]
        illegal_kwarg = [x for x in blacklist if x in kwargs]
        if illegal_kwarg:
            raise ValueError(
                f"Arguments {illegal_kwarg} cannot exist in kwargs, since FastEstimator will later directly use them in"
                " sklearn.metrics.confusion_matrix()")

check_kwargs staticmethod

Check if kwargs has any blacklist argument and raise an error if it does.

Parameters:

Name Type Description Default
kwargs Dict[str, Any]

Keywork arguments to be examined.

 required

Raises:

Type Description
ValueError

One of ["y_pred", "y_true", "labels"] argument exists in kwargs.
Source code in fastestimator/fastestimator/trace/metric/confusion_matrix.py 
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
@staticmethod
def check_kwargs(kwargs: Dict[str, Any]) -> None:
    """Check if `kwargs` has any blacklist argument and raise an error if it does.

    Args:
        kwargs: Keywork arguments to be examined.

    Raises:
        ValueError: One of ["y_pred", "y_true", "labels"] argument exists in `kwargs`.
    """
    blacklist = ["y_true", "y_pred", "labels"]
    illegal_kwarg = [x for x in blacklist if x in kwargs]
    if illegal_kwarg:
        raise ValueError(
            f"Arguments {illegal_kwarg} cannot exist in kwargs, since FastEstimator will later directly use them in"
            " sklearn.metrics.confusion_matrix()")