Source code for ivadomed.scripts.training_curve

#!/usr/bin/env python

import argparse
import numpy as np
from collections import defaultdict
import pandas as pd
import matplotlib.pyplot as plt
from textwrap import wrap
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
from ivadomed import utils as imed_utils
from pathlib import Path


def get_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("-i", "--input", required=True, type=str,
                        help="""Input path. If using --multiple, this parameter indicates
                                the prefix path of all log directories of interest. To compare
                                trainings (not using ``--multiple``) or set of trainings
                                (using ``--multiple``) with subplots, please list the paths by separating
                                them with commas, e.g. path_output1,path_output2.""",
                        metavar=imed_utils.Metavar.str)
    parser.add_argument("--multiple", required=False, dest="multiple", action='store_true',
                        help="""Multiple log directories are considered: all available folders
                                with -i as prefix. The plot represents the mean value (hard line)
                                surrounded by the standard deviation envelope.""")
    parser.add_argument("--lr", required=False, dest="learning_rate", action='store_true',
                        help="""Summary event file for learning rate is considered, the limits on
                                the y-axis plot are automatically defined.""")
    parser.add_argument("-y", "--ylim_loss", required=False, type=str,
                        help="""Indicates the limits on the y-axis for the loss plots, otherwise
                                these limits are automatically defined. Please separate the lower
                                and the upper limit by a comma, e.g. -1,0. Note: for the validation
                                metrics: the y-limits are always 0.0 and 1.0 except for the hausdorff
                                score where the limits are automatically defined.""",
                        metavar=imed_utils.Metavar.float)
    parser.add_argument("-o", "--output", required=True, type=str,
                        help="Output folder.", metavar=imed_utils.Metavar.file)
    return parser


def get_events_path_list(input_folder, learning_rate):
    """Check to make sure there is at most one summary event in any folder or any subfolder,
    and returns a list of summary event paths.

    A summary is defined as any file of the format ``events.out.tfevents.{...}```

    Args:
        input_folder (str): Input folder path.
        learning_rate (bool): Indicate if learning_rate is considered.
    Returns:
        list : a list of events paths
    """
    events_path_list = []

    # Check for events file in sub-folders
    for fold_path in Path(input_folder).iterdir():
        if fold_path.is_dir():
            event_list = [f.name for f in fold_path.iterdir() if f.name.startswith("events.out.tfevents.")]
            if len(event_list):
                if len(event_list) > 1:
                    raise ValueError(f"Multiple summary found in this folder: {fold_path}.\n"
                                     f"Please keep only one before running this script again.")
                else:
                    events_path_list.append(fold_path)
    # Sort events_path_list alphabetically
    events_path_list = sorted(events_path_list)

    if learning_rate:
    # Check for events file at the root of input_folder (contains learning_rate)
        event_list = [f.name for f in Path(input_folder).iterdir() if f.name.startswith("events.out.tfevents.")]
        if len(event_list):
            if len(event_list) > 1:
                raise ValueError(f"Multiple summary found in this folder: {Path(input_folder)}.\n"
                                 f"Please keep only one before running this script again.")
            else:
                # Append learning_rate events file at the end of events_path_list
                events_path_list.append(Path(input_folder))

    return events_path_list


def plot_curve(data_list, y_label, fig_ax, subplot_title, y_lim=None):
    """Plot curve of metrics or losses for each epoch.

    Args:
        data_list (list): list of pd.DataFrame, one for each path_output
        y_label (str): Label for the y-axis.
        fig_ax (plt.subplot):
        subplot_title (str): Title of the subplot
        y_lim (list): List of the lower and upper limits of the y-axis.
    """
    # Create count of the number of epochs
    max_nb_epoch = max([len(data_list[i]) for i in range(len(data_list))])
    epoch_count = range(1, max_nb_epoch + 1)

    for k in data_list[0].keys():
        data_k = pd.concat([data_list[i][k] for i in range(len(data_list))], axis=1)
        mean_data_k = data_k.mean(axis=1, skipna=True)
        std_data_k = data_k.std(axis=1, skipna=True)
        std_minus_data_k = (mean_data_k - std_data_k).tolist()
        std_plus_data_k = (mean_data_k + std_data_k).tolist()
        mean_data_k = mean_data_k.tolist()
        fig_ax.plot(epoch_count, mean_data_k, )
        fig_ax.fill_between(epoch_count, std_minus_data_k, std_plus_data_k, alpha=0.3)

    fig_ax.legend(data_list[0].keys(), loc="best")
    fig_ax.grid(linestyle='dotted')
    fig_ax.set_xlabel('Epoch')
    fig_ax.set_ylabel(y_label)
    if y_lim is not None:
        fig_ax.set_ylim(y_lim)
    fig_ax.set_xlim([1, max_nb_epoch])
    fig_ax.title.set_text('\n'.join(wrap(subplot_title, 80)))


[docs]def run_plot_training_curves(input_folder, output_folder, multiple_training=False, learning_rate=False, y_lim_loss=None): """Utility function to plot the training curves and save data as .csv files. This function uses the TensorFlow summary that is generated during a training to plot for each epoch: - the training against the validation loss, - the metrics computed on the validation sub-dataset, - the learning rate if learning_rate is True. It could consider one output path at a time, for example: .. image:: https://raw.githubusercontent.com/ivadomed/doc-figures/main/scripts/plot_loss_single.png :width: 600px :align: center ... or multiple (using ``multiple_training=True``). In that case, the hard line represents the mean value across the trainings whereas the envelope represents the standard deviation: .. image:: https://raw.githubusercontent.com/ivadomed/doc-figures/main/scripts/plot_loss_multiple.png :width: 600px :align: center It is also possible to compare multiple trainings (or set of trainings) by listing them in ``-i``, separated by commas: .. image:: https://raw.githubusercontent.com/ivadomed/doc-figures/main/scripts/plot_loss_mosaic.png :width: 600px :align: center Args: input_folder (str): Input path name. Flag: ``--input``, ``-i``. If using ``--multiple``, this parameter indicates the prefix path of all log directories of interest. To compare trainings (not using ``--multiple``) or set of trainings (using ``--multiple``) with subplots, please list the paths by separating them with commas, e.g. path_output1, path_output2 output_folder (str): Output folder. Flag: ``--output``, ``-o``. multiple_training (bool): Indicates if multiple log directories are considered (``True``) or not (``False``). Flag: ``--multiple``. All available folders with ``-i`` as prefix are considered. The plot represents the mean value (hard line) surrounded by the standard deviation (envelope). learning_rate (bool): Indicates if the summary event file for learning rate is considered (``True``) or not (``False``). Flag: ``--lr``. The limits on the y-axis plot are automatically defined. y_lim_loss (list): List of the lower and upper limits of the y-axis of the loss plot, otherwise these limits are automatically defined. Please separate the lower and the upper limit by a comma, e.g. -1,0. Note: for the validation metrics: the y-limits are always 0.0 and 1.0 except for the hausdorff score where the limits are automatically defined. """ group_list = input_folder.split(",") plt_dict = {} # Create output folder if Path(output_folder).is_dir(): print(f"Output folder already exists: {output_folder}.") else: print(f"Creating output folder: {output_folder}.") Path(output_folder).mkdir(parents=True) # Config subplots if len(group_list) > 1: n_cols = 2 n_rows = int(np.ceil(len(group_list) / float(n_cols))) else: n_cols, n_rows = 1, 1 for i_subplot, input_folder in enumerate(group_list): input_folder = Path(input_folder).expanduser() # Find training folders: if multiple_training: prefix = input_folder.name input_folder = input_folder.parent input_folder_list = [f for f in input_folder.iterdir() if f.name.startswith(prefix)] else: prefix = input_folder.name input_folder_list = [input_folder] events_df_list = [] for path_output in input_folder_list: # Find tf folders events_path_list = get_events_path_list(str(path_output), learning_rate) # Get data as dataframe and save as .csv file events_vals_df = tensorboard_retrieve_event(events_path_list) events_vals_df.to_csv(Path(output_folder, str(path_output.name) + "_training_values.csv")) # Store data events_df_list.append(events_vals_df) # Plot train and valid losses together loss_keys = [k for k in events_df_list[0].keys() if k.endswith("loss")] if i_subplot == 0: # Init plot plt_dict[str(Path(output_folder, "losses.png"))] = plt.figure(figsize=(10 * n_cols, 5 * n_rows)) ax = plt_dict[str(Path(output_folder, "losses.png"))].add_subplot(n_rows, n_cols, i_subplot + 1) plot_curve([df[loss_keys] for df in events_df_list], y_label="loss", fig_ax=ax, subplot_title=prefix, y_lim=y_lim_loss) # Plot each validation metric and learning rate separately for tag in events_df_list[0].keys(): if not tag.endswith("loss"): if i_subplot == 0: # Init plot plt_dict[str(Path(output_folder, tag + ".png"))] = plt.figure(figsize=(10 * n_cols, 5 * n_rows)) ax = plt_dict[str(Path(output_folder, tag + ".png"))].add_subplot(n_rows, n_cols, i_subplot + 1) y_lim = None if (tag.startswith("hausdorff") or tag.startswith("learning_rate")) else [0, 1] plot_curve(data_list=[df[[tag]] for df in events_df_list], y_label=tag, fig_ax=ax, subplot_title=prefix, y_lim=y_lim) for fname_out in plt_dict: plt_dict[fname_out].savefig(fname_out)
def tensorboard_retrieve_event(events_path_list): """Retrieve data from tensorboard summary event. Args: events_path_list (list): list of events paths Returns: df: a panda dataframe where the columns are the metric or loss and the row are the epochs. """ # Lists of metrics and losses in the same order as in events_path_list list_metrics = [] list_loss = [] for events in events_path_list: if str(events.name).startswith("Validation_Metrics_"): metric_name = str(events.name.split("Validation_Metrics_")[1]) list_metrics.append(metric_name) elif str(events.name).startswith("losses_"): loss_name = str(events.name.split("losses_")[1]) list_loss.append(loss_name) # Each element in the summary iterator represent an element (e.g., scalars, images..) # stored in the summary for all epochs in the form of event, in the same order as in events_path_list. summary_iterators = [EventAccumulator(str(events)).Reload() for events in events_path_list] metrics = defaultdict(list) num_metrics = 0 num_loss = 0 num_lr = 0 for i in range(len(summary_iterators)): if summary_iterators[i].Tags()['scalars'] == ['Validation/Metrics']: # we create a empty list out = [0 for i in range(len(summary_iterators[i].Scalars("Validation/Metrics")))] # we ensure that value are append in the right order by looking at the step value # (which represents the epoch) for events in summary_iterators[i].Scalars("Validation/Metrics"): out[events.step - 1] = events.value # keys are the defined metrics metrics[list_metrics[num_metrics]] = out num_metrics += 1 elif summary_iterators[i].Tags()['scalars'] == ['losses']: out = [0 for i in range(len(summary_iterators[i].Scalars("losses")))] # we ensure that value are append in the right order by looking at the step value # (which represents the epoch) for events in summary_iterators[i].Scalars("losses"): out[events.step - 1] = events.value metrics[list_loss[num_loss]] = out num_loss += 1 elif summary_iterators[i].Tags()['scalars'] == ['learning_rate']: out = [0 for i in range(len(summary_iterators[i].Scalars("learning_rate")))] for events in summary_iterators[i].Scalars("learning_rate"): out[events.step - 1] = events.value metrics['learning_rate'] = out num_lr += 1 if num_loss == 0 and num_metrics == 0 and num_lr == 0: raise Exception('No metrics, losses or learning rate found in the event') metrics_df = pd.DataFrame.from_dict(metrics) return metrics_df def main(args=None): imed_utils.init_ivadomed() parser = get_parser() args = imed_utils.get_arguments(parser, args) y_lim_loss = [int(y) for y in args.ylim_loss.split(',')] if args.ylim_loss else None run_plot_training_curves(input_folder=args.input, output_folder=args.output, multiple_training=args.multiple, learning_rate=args.learning_rate, y_lim_loss=y_lim_loss) if __name__ == '__main__': main()