We'll start by getting the imports out of the way:
InĀ [1]:
Copied!
import tempfile
import os
import fastestimator as fe
from fastestimator.architecture.tensorflow import LeNet
from fastestimator.backend import squeeze, reduce_mean
from fastestimator.dataset.data import cifair10
from fastestimator.op.numpyop.meta import Sometimes
from fastestimator.op.numpyop.multivariate import HorizontalFlip, PadIfNeeded, RandomCrop
from fastestimator.op.numpyop.univariate import CoarseDropout, Normalize
from fastestimator.op.tensorop.loss import CrossEntropy
from fastestimator.op.tensorop.model import ModelOp, UpdateOp
from fastestimator.op.tensorop import LambdaOp
from fastestimator.schedule import RepeatScheduler
from fastestimator.trace.adapt import LRScheduler
from fastestimator.trace.io import BestModelSaver, ImageViewer, TensorBoard
from fastestimator.trace.metric import MCC
from fastestimator.trace.xai import LabelTracker
from fastestimator.util import to_number
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Conv2D, MaxPooling2D, Dense, Flatten, Dropout, BatchNormalization
import matplotlib.pyplot as plt
import numpy as np
label_mapping = {
'airplane': 0,
'automobile': 1,
'bird': 2,
'cat': 3,
'deer': 4,
'dog': 5,
'frog': 6,
'horse': 7,
'ship': 8,
'truck': 9
}
import tempfile
import os
import fastestimator as fe
from fastestimator.architecture.tensorflow import LeNet
from fastestimator.backend import squeeze, reduce_mean
from fastestimator.dataset.data import cifair10
from fastestimator.op.numpyop.meta import Sometimes
from fastestimator.op.numpyop.multivariate import HorizontalFlip, PadIfNeeded, RandomCrop
from fastestimator.op.numpyop.univariate import CoarseDropout, Normalize
from fastestimator.op.tensorop.loss import CrossEntropy
from fastestimator.op.tensorop.model import ModelOp, UpdateOp
from fastestimator.op.tensorop import LambdaOp
from fastestimator.schedule import RepeatScheduler
from fastestimator.trace.adapt import LRScheduler
from fastestimator.trace.io import BestModelSaver, ImageViewer, TensorBoard
from fastestimator.trace.metric import MCC
from fastestimator.trace.xai import LabelTracker
from fastestimator.util import to_number
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Conv2D, MaxPooling2D, Dense, Flatten, Dropout, BatchNormalization
import matplotlib.pyplot as plt
import numpy as np
label_mapping = {
'airplane': 0,
'automobile': 1,
'bird': 2,
'cat': 3,
'deer': 4,
'dog': 5,
'frog': 6,
'horse': 7,
'ship': 8,
'truck': 9
}
Label Tracking¶
Suppose you are doing some training, and you want to know whether a particular class is easier or harder than other classes for your network to learn. One way to investigate this is with the LabelTracker
Trace
. It takes as input any per-element metric (such as sample-wise loss), as well as any label vector (usually class labels, but it could be any grouping) and produces a visualization at the end of training:
InĀ [2]:
Copied!
batch_size=128
save_dir = tempfile.mkdtemp()
train_data, eval_data = cifair10.load_data()
test_data = eval_data.split(range(len(eval_data) // 2))
pipeline = fe.Pipeline(
train_data=train_data,
eval_data=eval_data,
test_data=test_data,
batch_size=batch_size,
ops=[Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616)),
PadIfNeeded(min_height=40, min_width=40, image_in="x", image_out="x", mode="train"),
RandomCrop(32, 32, image_in="x", image_out="x", mode="train"),
Sometimes(HorizontalFlip(image_in="x", image_out="x", mode="train")),
CoarseDropout(inputs="x", outputs="x", mode="train", max_holes=1),
],
num_process=0)
model = fe.build(model_fn=lambda: LeNet(input_shape=(32, 32, 3)), optimizer_fn="adam")
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred"),
CrossEntropy(inputs=("y_pred", "y"), outputs="ce", mode="train"),
CrossEntropy(inputs=("y_pred", "y"), outputs="sample_ce", mode=("eval", "test"), average_loss=False),
LambdaOp(inputs="sample_ce", outputs="ce", mode=("eval", "test"), fn=lambda x: reduce_mean(x)),
UpdateOp(model=model, loss_name="ce")
])
traces = [
MCC(true_key="y", pred_key="y_pred"),
BestModelSaver(model=model, save_dir=save_dir, metric="mcc", save_best_mode="max", load_best_final=True),
LabelTracker(label="y", metric="sample_ce", label_mapping=label_mapping, outputs="ce_vs_y", bounds=None, mode=["eval", "test"]),
ImageViewer(inputs="ce_vs_y", mode=["eval", "test"])
]
estimator = fe.Estimator(pipeline=pipeline,
network=network,
epochs=10,
traces=traces,
log_steps=300)
batch_size=128
save_dir = tempfile.mkdtemp()
train_data, eval_data = cifair10.load_data()
test_data = eval_data.split(range(len(eval_data) // 2))
pipeline = fe.Pipeline(
train_data=train_data,
eval_data=eval_data,
test_data=test_data,
batch_size=batch_size,
ops=[Normalize(inputs="x", outputs="x", mean=(0.4914, 0.4822, 0.4465), std=(0.2471, 0.2435, 0.2616)),
PadIfNeeded(min_height=40, min_width=40, image_in="x", image_out="x", mode="train"),
RandomCrop(32, 32, image_in="x", image_out="x", mode="train"),
Sometimes(HorizontalFlip(image_in="x", image_out="x", mode="train")),
CoarseDropout(inputs="x", outputs="x", mode="train", max_holes=1),
],
num_process=0)
model = fe.build(model_fn=lambda: LeNet(input_shape=(32, 32, 3)), optimizer_fn="adam")
network = fe.Network(ops=[
ModelOp(model=model, inputs="x", outputs="y_pred"),
CrossEntropy(inputs=("y_pred", "y"), outputs="ce", mode="train"),
CrossEntropy(inputs=("y_pred", "y"), outputs="sample_ce", mode=("eval", "test"), average_loss=False),
LambdaOp(inputs="sample_ce", outputs="ce", mode=("eval", "test"), fn=lambda x: reduce_mean(x)),
UpdateOp(model=model, loss_name="ce")
])
traces = [
MCC(true_key="y", pred_key="y_pred"),
BestModelSaver(model=model, save_dir=save_dir, metric="mcc", save_best_mode="max", load_best_final=True),
LabelTracker(label="y", metric="sample_ce", label_mapping=label_mapping, outputs="ce_vs_y", bounds=None, mode=["eval", "test"]),
ImageViewer(inputs="ce_vs_y", mode=["eval", "test"])
]
estimator = fe.Estimator(pipeline=pipeline,
network=network,
epochs=10,
traces=traces,
log_steps=300)
InĀ [3]:
Copied!
estimator.fit()
estimator.fit()
______ __ ______ __ _ __ / ____/___ ______/ /_/ ____/____/ /_(_)___ ___ ____ _/ /_____ _____ / /_ / __ `/ ___/ __/ __/ / ___/ __/ / __ `__ \/ __ `/ __/ __ \/ ___/ / __/ / /_/ (__ ) /_/ /___(__ ) /_/ / / / / / / /_/ / /_/ /_/ / / /_/ \__,_/____/\__/_____/____/\__/_/_/ /_/ /_/\__,_/\__/\____/_/ FastEstimator-Start: step: 1; logging_interval: 300; num_device: 0; FastEstimator-Train: step: 1; ce: 2.32721; FastEstimator-Train: step: 300; ce: 1.338028; steps/sec: 14.89; FastEstimator-Train: step: 391; epoch: 1; epoch_time: 27.9 sec; FastEstimator-BestModelSaver: Saved model to /var/folders/3r/h9kh47050gv6rbt_pgf8cl540000gn/T/tmpdcv5kfby/model_best_mcc.h5 FastEstimator-Eval: step: 391; epoch: 1; ce: 1.2785366; max_mcc: 0.48404556307705315; mcc: 0.48404556307705315; since_best_mcc: 0; FastEstimator-Train: step: 600; ce: 1.2232394; steps/sec: 14.02; FastEstimator-Train: step: 782; epoch: 2; epoch_time: 29.12 sec; FastEstimator-BestModelSaver: Saved model to /var/folders/3r/h9kh47050gv6rbt_pgf8cl540000gn/T/tmpdcv5kfby/model_best_mcc.h5 FastEstimator-Eval: step: 782; epoch: 2; ce: 1.1373289; max_mcc: 0.5609728402773733; mcc: 0.5609728402773733; since_best_mcc: 0; FastEstimator-Train: step: 900; ce: 1.1405166; steps/sec: 13.13; FastEstimator-Train: step: 1173; epoch: 3; epoch_time: 29.1 sec; FastEstimator-BestModelSaver: Saved model to /var/folders/3r/h9kh47050gv6rbt_pgf8cl540000gn/T/tmpdcv5kfby/model_best_mcc.h5 FastEstimator-Eval: step: 1173; epoch: 3; ce: 1.0307672; max_mcc: 0.6016765128859065; mcc: 0.6016765128859065; since_best_mcc: 0; FastEstimator-Train: step: 1200; ce: 1.1333961; steps/sec: 13.31; FastEstimator-Train: step: 1500; ce: 1.0101137; steps/sec: 12.4; FastEstimator-Train: step: 1564; epoch: 4; epoch_time: 31.63 sec; FastEstimator-BestModelSaver: Saved model to /var/folders/3r/h9kh47050gv6rbt_pgf8cl540000gn/T/tmpdcv5kfby/model_best_mcc.h5 FastEstimator-Eval: step: 1564; epoch: 4; ce: 0.97532713; max_mcc: 0.6108125458368492; mcc: 0.6108125458368492; since_best_mcc: 0; FastEstimator-Train: step: 1800; ce: 1.1236746; steps/sec: 11.66; FastEstimator-Train: step: 1955; epoch: 5; epoch_time: 32.6 sec; FastEstimator-BestModelSaver: Saved model to /var/folders/3r/h9kh47050gv6rbt_pgf8cl540000gn/T/tmpdcv5kfby/model_best_mcc.h5 FastEstimator-Eval: step: 1955; epoch: 5; ce: 0.9481665; max_mcc: 0.6256908330205657; mcc: 0.6256908330205657; since_best_mcc: 0; FastEstimator-Train: step: 2100; ce: 1.1856788; steps/sec: 12.55; FastEstimator-Train: step: 2346; epoch: 6; epoch_time: 32.38 sec; FastEstimator-BestModelSaver: Saved model to /var/folders/3r/h9kh47050gv6rbt_pgf8cl540000gn/T/tmpdcv5kfby/model_best_mcc.h5 FastEstimator-Eval: step: 2346; epoch: 6; ce: 0.87894356; max_mcc: 0.6538663972122967; mcc: 0.6538663972122967; since_best_mcc: 0; FastEstimator-Train: step: 2400; ce: 1.0800145; steps/sec: 11.93; FastEstimator-Train: step: 2700; ce: 0.9510909; steps/sec: 11.85; FastEstimator-Train: step: 2737; epoch: 7; epoch_time: 32.63 sec; FastEstimator-BestModelSaver: Saved model to /var/folders/3r/h9kh47050gv6rbt_pgf8cl540000gn/T/tmpdcv5kfby/model_best_mcc.h5 FastEstimator-Eval: step: 2737; epoch: 7; ce: 0.85947037; max_mcc: 0.6650824675644685; mcc: 0.6650824675644685; since_best_mcc: 0; FastEstimator-Train: step: 3000; ce: 0.7190244; steps/sec: 11.88; FastEstimator-Train: step: 3128; epoch: 8; epoch_time: 32.49 sec; FastEstimator-BestModelSaver: Saved model to /var/folders/3r/h9kh47050gv6rbt_pgf8cl540000gn/T/tmpdcv5kfby/model_best_mcc.h5 FastEstimator-Eval: step: 3128; epoch: 8; ce: 0.8448655; max_mcc: 0.6689255911737415; mcc: 0.6689255911737415; since_best_mcc: 0; FastEstimator-Train: step: 3300; ce: 0.967314; steps/sec: 12.46; FastEstimator-Train: step: 3519; epoch: 9; epoch_time: 31.84 sec; FastEstimator-BestModelSaver: Saved model to /var/folders/3r/h9kh47050gv6rbt_pgf8cl540000gn/T/tmpdcv5kfby/model_best_mcc.h5 FastEstimator-Eval: step: 3519; epoch: 9; ce: 0.8158425; max_mcc: 0.6860621181942419; mcc: 0.6860621181942419; since_best_mcc: 0; FastEstimator-Train: step: 3600; ce: 1.018165; steps/sec: 12.3; FastEstimator-Train: step: 3900; ce: 0.9490619; steps/sec: 10.65; FastEstimator-Train: step: 3910; epoch: 10; epoch_time: 35.45 sec; FastEstimator-Eval: step: 3910; epoch: 10; ce: 0.8028839; max_mcc: 0.6860621181942419; mcc: 0.6860295312279335; since_best_mcc: 1; FastEstimator-BestModelSaver: Restoring model from /var/folders/3r/h9kh47050gv6rbt_pgf8cl540000gn/T/tmpdcv5kfby/model_best_mcc.h5
FastEstimator-Finish: step: 3910; model_lr: 0.001; total_time: 329.01 sec;
From the graph above it seems that cats are relatively difficult for the network to learn well, whereas automobiles are pretty easy.