'''Runs error analysis on score sets, outputs metrics and plots'''
from __future__ import division, print_function
from abc import ABCMeta, abstractmethod
import math
import sys
import os.path
import numpy
import click
import matplotlib
import matplotlib.pyplot as mpl
from matplotlib.backends.backend_pdf import PdfPages
from tabulate import tabulate
from .. import (far_threshold, plot, utils, ppndf)
import logging
LOGGER = logging.getLogger("bob.measure")
[docs]def check_list_value(values, desired_number, name, name2='systems'):
if values is not None and len(values) != desired_number:
if len(values) == 1:
values = values * desired_number
else:
raise click.BadParameter(
'#{} ({}) must be either 1 value or the same as '
'#{} ({} values)'.format(name, values, name2, desired_number))
return values
[docs]class MeasureBase(object):
"""Base class for metrics and plots.
This abstract class define the framework to plot or compute metrics from a
list of (positive, negative) scores tuples.
Attributes
----------
func_load:
Function that is used to load the input files
"""
__metaclass__ = ABCMeta # for python 2.7 compatibility
def __init__(self, ctx, scores, evaluation, func_load):
"""
Parameters
----------
ctx : :py:class:`dict`
Click context dictionary.
scores : :any:`list`:
List of input files (e.g. dev-{1, 2, 3}, {dev,eval}-scores1
{dev,eval}-scores2)
eval : :py:class:`bool`
True if eval data are used
func_load : Function that is used to load the input files
"""
self._scores = scores
self._ctx = ctx
self.func_load = func_load
self._legends = ctx.meta.get('legends')
self._eval = evaluation
self._min_arg = ctx.meta.get('min_arg', 1)
if len(scores) < 1 or len(scores) % self._min_arg != 0:
raise click.BadParameter(
'Number of argument must be a non-zero multiple of %d' % self._min_arg
)
self.n_systems = int(len(scores) / self._min_arg)
if self._legends is not None and len(self._legends) < self.n_systems:
raise click.BadParameter("Number of legends must be >= to the "
"number of systems")
[docs] def run(self):
""" Generate outputs (e.g. metrics, files, pdf plots).
This function calls abstract methods
:func:`~bob.measure.script.figure.MeasureBase.init_process` (before
loop), :py:func:`~bob.measure.script.figure.MeasureBase.compute`
(in the loop iterating through the different
systems) and :py:func:`~bob.measure.script.figure.MeasureBase.end_process`
(after the loop).
"""
# init matplotlib, log files, ...
self.init_process()
# iterates through the different systems and feed `compute`
# with the dev (and eval) scores of each system
# Note that more than one dev or eval scores score can be passed to
# each system
for idx in range(self.n_systems):
# load scores for each system: get the corresponding arrays and
# base-name of files
input_scores, input_names = self._load_files(
# Scores are given as followed:
# SysA-dev SysA-eval ... SysA-XX SysB-dev SysB-eval ... SysB-XX
# ------------------------------ ------------------------------
# First set of `self._min_arg` Second set of input files
# input files starting at for SysB
# index idx * self._min_arg
self._scores[idx * self._min_arg:(idx + 1) * self._min_arg]
)
LOGGER.info("-----Input files for system %d-----", idx + 1)
for i, name in enumerate(input_names):
if not self._eval:
LOGGER.info("Dev. score %d: %s", i + 1, name)
else:
if i % 2 == 0:
LOGGER.info("Dev. score %d: %s", i / 2 + 1, name)
else:
LOGGER.info("Eval. score %d: %s", i / 2 + 1, name)
LOGGER.info("----------------------------------")
self.compute(idx, input_scores, input_names)
# setup final configuration, plotting properties, ...
self.end_process()
# protected functions that need to be overwritten
[docs] def init_process(self):
""" Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
before iterating through the different systems.
Should reimplemented in derived classes"""
pass
# Main computations are done here in the subclasses
[docs] @abstractmethod
def compute(self, idx, input_scores, input_names):
"""Compute metrics or plots from the given scores provided by
:py:func:`~bob.measure.script.figure.MeasureBase.run`.
Should reimplemented in derived classes
Parameters
----------
idx : :obj:`int`
index of the system
input_scores: :any:`list`
list of scores returned by the loading function
input_names: :any:`list`
list of base names for the input file of the system
"""
pass
# structure of input is (vuln example):
# if evaluation is provided
# [ (dev_licit_neg, dev_licit_pos), (eval_licit_neg, eval_licit_pos),
# (dev_spoof_neg, dev_licit_pos), (eval_spoof_neg, eval_licit_pos)]
# and if only dev:
# [ (dev_licit_neg, dev_licit_pos), (dev_spoof_neg, dev_licit_pos)]
# Things to do after the main iterative computations are done
[docs] @abstractmethod
def end_process(self):
""" Called in :py:func:`~bob.measure.script.figure.MeasureBase`.run
after iterating through the different systems.
Should reimplemented in derived classes"""
pass
# common protected functions
def _load_files(self, filepaths):
''' Load the input files and return the base names of the files
Returns
-------
scores: :any:`list`:
A list that contains the output of
``func_load`` for the given files
basenames: :any:`list`:
A list of the given files
'''
scores = []
basenames = []
for filename in filepaths:
basenames.append(filename)
scores.append(self.func_load(filename))
return scores, basenames
[docs]class Metrics(MeasureBase):
''' Compute metrics from score files
Attributes
----------
log_file: str
output stream
'''
def __init__(self, ctx, scores, evaluation, func_load,
names=('False Positive Rate', 'False Negative Rate',
'Precision', 'Recall', 'F1-score')):
super(Metrics, self).__init__(ctx, scores, evaluation, func_load)
self.names = names
self._tablefmt = ctx.meta.get('tablefmt')
self._criterion = ctx.meta.get('criterion')
self._open_mode = ctx.meta.get('open_mode')
self._thres = ctx.meta.get('thres')
self._decimal = ctx.meta.get('decimal', 2)
if self._thres is not None:
if len(self._thres) == 1:
self._thres = self._thres * self.n_systems
elif len(self._thres) != self.n_systems:
raise click.BadParameter(
'#thresholds must be the same as #systems (%d)'
% len(self.n_systems)
)
self._far = ctx.meta.get('far_value')
self._log = ctx.meta.get('log')
self.log_file = sys.stdout
if self._log is not None:
self.log_file = open(self._log, self._open_mode)
[docs] def get_thres(self, criterion, dev_neg, dev_pos, far):
return utils.get_thres(criterion, dev_neg, dev_pos, far)
def _numbers(self, neg, pos, threshold, fta):
from .. import (farfrr, precision_recall, f_score)
# fpr and fnr
fmr, fnmr = farfrr(neg, pos, threshold)
hter = (fmr + fnmr) / 2.0
far = fmr * (1 - fta)
frr = fta + fnmr * (1 - fta)
ni = neg.shape[0] # number of impostors
fm = int(round(fmr * ni)) # number of false accepts
nc = pos.shape[0] # number of clients
fnm = int(round(fnmr * nc)) # number of false rejects
# precision and recall
precision, recall = precision_recall(neg, pos, threshold)
# f_score
f1_score = f_score(neg, pos, threshold, 1)
return (fta, fmr, fnmr, hter, far, frr, fm, ni, fnm, nc, precision,
recall, f1_score)
def _strings(self, metrics):
n_dec = '.%df' % self._decimal
fta_str = "%s%%" % format(100 * metrics[0], n_dec)
fmr_str = "%s%% (%d/%d)" % (format(100 * metrics[1], n_dec),
metrics[6], metrics[7])
fnmr_str = "%s%% (%d/%d)" % (format(100 * metrics[2], n_dec),
metrics[8], metrics[9])
far_str = "%s%%" % format(100 * metrics[4], n_dec)
frr_str = "%s%%" % format(100 * metrics[5], n_dec)
hter_str = "%s%%" % format(100 * metrics[3], n_dec)
prec_str = "%s" % format(metrics[10], n_dec)
recall_str = "%s" % format(metrics[11], n_dec)
f1_str = "%s" % format(metrics[12], n_dec)
return (fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str,
prec_str, recall_str, f1_str)
def _get_all_metrics(self, idx, input_scores, input_names):
''' Compute all metrics for dev and eval scores'''
neg_list, pos_list, fta_list = utils.get_fta_list(input_scores)
dev_neg, dev_pos, dev_fta = neg_list[0], pos_list[0], fta_list[0]
dev_file = input_names[0]
if self._eval:
eval_neg, eval_pos, eval_fta = neg_list[1], pos_list[1], fta_list[1]
threshold = self.get_thres(self._criterion, dev_neg, dev_pos, self._far) \
if self._thres is None else self._thres[idx]
title = self._legends[idx] if self._legends is not None else None
if self._thres is None:
far_str = ''
if self._criterion == 'far' and self._far is not None:
far_str = str(self._far)
click.echo("[Min. criterion: %s %s] Threshold on Development set `%s`: %e"
% (self._criterion.upper(),
far_str, title or dev_file,
threshold),
file=self.log_file)
else:
click.echo("[Min. criterion: user provided] Threshold on "
"Development set `%s`: %e"
% (dev_file or title, threshold), file=self.log_file)
res = []
res.append(self._strings(self._numbers(
dev_neg, dev_pos, threshold, dev_fta)))
if self._eval:
# computes statistics for the eval set based on the threshold a
# priori
res.append(self._strings(self._numbers(
eval_neg, eval_pos, threshold, eval_fta)))
else:
res.append(None)
return res
[docs] def compute(self, idx, input_scores, input_names):
''' Compute metrics thresholds and tables (FPR, FNR, precision, recall,
f1_score) for given system inputs'''
dev_file = input_names[0]
title = self._legends[idx] if self._legends is not None else None
all_metrics = self._get_all_metrics(idx, input_scores, input_names)
fta_dev = float(all_metrics[0][0].replace('%', ''))
if fta_dev > 0.0:
LOGGER.warn("NaNs scores (%s) were found in %s amd removed",
all_metrics[0][0], dev_file)
headers = [' ' or title, 'Development']
rows = [[self.names[0], all_metrics[0][1]],
[self.names[1], all_metrics[0][2]],
[self.names[2], all_metrics[0][6]],
[self.names[3], all_metrics[0][7]],
[self.names[4], all_metrics[0][8]]]
if self._eval:
eval_file = input_names[1]
fta_eval = float(all_metrics[1][0].replace('%', ''))
if fta_eval > 0.0:
LOGGER.warn("NaNs scores (%s) were found in %s and removed.",
all_metrics[1][0], eval_file)
# computes statistics for the eval set based on the threshold a
# priori
headers.append('Evaluation')
rows[0].append(all_metrics[1][1])
rows[1].append(all_metrics[1][2])
rows[2].append(all_metrics[1][6])
rows[3].append(all_metrics[1][7])
rows[4].append(all_metrics[1][8])
click.echo(tabulate(rows, headers, self._tablefmt), file=self.log_file)
[docs] def end_process(self):
''' Close log file if needed'''
if self._log is not None:
self.log_file.close()
[docs]class MultiMetrics(Metrics):
'''Computes average of metrics based on several protocols (cross
validation)
Attributes
----------
log_file : str
output stream
names : tuple
List of names for the metrics.
'''
def __init__(self, ctx, scores, evaluation, func_load,
names=('NaNs Rate', 'False Positive Rate',
'False Negative Rate', 'False Accept Rate',
'False Reject Rate', 'Half Total Error Rate')):
super(MultiMetrics, self).__init__(
ctx, scores, evaluation, func_load, names=names)
self.headers = ['Methods'] + list(self.names)
if self._eval:
self.headers.insert(1, self.names[5] + ' (dev)')
self.rows = []
def _strings(self, metrics):
ftam, fmrm, fnmrm, hterm, farm, frrm, _, _, _, _, _, _, _ = \
metrics.mean(axis=0)
ftas, fmrs, fnmrs, hters, fars, frrs, _, _, _, _, _, _, _ = \
metrics.std(axis=0)
n_dec = '.%df' % self._decimal
fta_str = "%s%% (%s%%)" % (format(100 * ftam, n_dec),
format(100 * ftas, n_dec))
fmr_str = "%s%% (%s%%)" % (format(100 * fmrm, n_dec),
format(100 * fmrs, n_dec))
fnmr_str = "%s%% (%s%%)" % (format(100 * fnmrm, n_dec),
format(100 * fnmrs, n_dec))
far_str = "%s%% (%s%%)" % (format(100 * farm, n_dec),
format(100 * fars, n_dec))
frr_str = "%s%% (%s%%)" % (format(100 * frrm, n_dec),
format(100 * frrs, n_dec))
hter_str = "%s%% (%s%%)" % (format(100 * hterm, n_dec),
format(100 * hters, n_dec))
return fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str
[docs] def compute(self, idx, input_scores, input_names):
'''Computes the average of metrics over several protocols.'''
neg_list, pos_list, fta_list = utils.get_fta_list(input_scores)
step = 2 if self._eval else 1
self._dev_metrics = []
self._thresholds = []
for i in range(0, len(input_scores), step):
neg, pos, fta = neg_list[i], pos_list[i], fta_list[i]
threshold = self.get_thres(self._criterion, neg, pos, self._far) \
if self._thres is None else self._thres[idx]
self._thresholds.append(threshold)
self._dev_metrics.append(self._numbers(neg, pos, threshold, fta))
self._dev_metrics = numpy.array(self._dev_metrics)
if self._eval:
self._eval_metrics = []
for i in range(1, len(input_scores), step):
neg, pos, fta = neg_list[i], pos_list[i], fta_list[i]
threshold = self._thresholds[i // 2]
self._eval_metrics.append(
self._numbers(neg, pos, threshold, fta))
self._eval_metrics = numpy.array(self._eval_metrics)
title = self._legends[idx] if self._legends is not None else None
fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str = \
self._strings(self._dev_metrics)
if self._eval:
self.rows.append([title, hter_str])
else:
self.rows.append([title, fta_str, fmr_str, fnmr_str,
far_str, frr_str, hter_str])
if self._eval:
# computes statistics for the eval set based on the threshold a
# priori
fta_str, fmr_str, fnmr_str, far_str, frr_str, hter_str = \
self._strings(self._eval_metrics)
self.rows[-1].extend([fta_str, fmr_str, fnmr_str,
far_str, frr_str, hter_str])
[docs] def end_process(self):
click.echo(tabulate(self.rows, self.headers,
self._tablefmt), file=self.log_file)
super(MultiMetrics, self).end_process()
[docs]class PlotBase(MeasureBase):
''' Base class for plots. Regroup several options and code
shared by the different plots
'''
def __init__(self, ctx, scores, evaluation, func_load):
super(PlotBase, self).__init__(ctx, scores, evaluation, func_load)
self._output = ctx.meta.get('output')
self._points = ctx.meta.get('points', 100)
self._split = ctx.meta.get('split')
self._axlim = ctx.meta.get('axlim')
self._disp_legend = ctx.meta.get('disp_legend', True)
self._legend_loc = ctx.meta.get('legend_loc')
self._min_dig = None
if 'min_far_value' in ctx.meta:
self._min_dig = int(math.log10(ctx.meta['min_far_value']))
elif self._axlim is not None and self._axlim[0] is not None:
self._min_dig = int(math.log10(self._axlim[0])
if self._axlim[0] != 0 else 0)
self._clayout = ctx.meta.get('clayout')
self._far_at = ctx.meta.get('lines_at')
self._trans_far_val = self._far_at
if self._far_at is not None:
self._eval_points = {line: [] for line in self._far_at}
self._lines_val = []
self._print_fn = ctx.meta.get('show_fn', True)
self._x_rotation = ctx.meta.get('x_rotation')
if 'style' in ctx.meta:
mpl.style.use(ctx.meta['style'])
self._nb_figs = 2 if self._eval and self._split else 1
self._colors = utils.get_colors(self.n_systems)
self._line_linestyles = ctx.meta.get('line_linestyles', False)
self._linestyles = utils.get_linestyles(
self.n_systems, self._line_linestyles)
self._titles = ctx.meta.get('titles', []) * 2
# for compatibility
self._title = ctx.meta.get('title')
if not self._titles and self._title is not None:
self._titles = [self._title] * 2
self._x_label = ctx.meta.get('x_label')
self._y_label = ctx.meta.get('y_label')
self._grid_color = 'silver'
self._pdf_page = None
self._end_setup_plot = True
[docs] def init_process(self):
''' Open pdf and set axis font size if provided '''
if not hasattr(matplotlib, 'backends'):
matplotlib.use('pdf')
self._pdf_page = self._ctx.meta['PdfPages'] if 'PdfPages'in \
self._ctx.meta else PdfPages(self._output)
for i in range(self._nb_figs):
fs = self._ctx.meta.get('figsize')
fig = mpl.figure(i + 1, figsize=fs)
fig.set_constrained_layout(self._clayout)
fig.clear()
[docs] def end_process(self):
''' Set title, legend, axis labels, grid colors, save figures, drow
lines and close pdf if needed '''
# draw vertical lines
if self._far_at is not None:
for (line, line_trans) in zip(self._far_at, self._trans_far_val):
mpl.figure(1)
mpl.plot(
[line_trans, line_trans], [-100.0, 100.], "--",
color='black'
)
if self._eval and self._split:
mpl.figure(2)
x_values = [i for i, _ in self._eval_points[line]]
y_values = [j for _, j in self._eval_points[line]]
sort_indice = sorted(
range(len(x_values)), key=x_values.__getitem__
)
x_values = [x_values[i] for i in sort_indice]
y_values = [y_values[i] for i in sort_indice]
mpl.plot(x_values,
y_values, '--',
color='black')
# only for plots
if self._end_setup_plot:
for i in range(self._nb_figs):
fig = mpl.figure(i + 1)
title = '' if not self._titles else self._titles[i]
mpl.title(title if title.replace(' ', '') else '')
mpl.xlabel(self._x_label)
mpl.ylabel(self._y_label)
mpl.grid(True, color=self._grid_color)
if self._disp_legend:
mpl.legend(loc=self._legend_loc)
self._set_axis()
mpl.xticks(rotation=self._x_rotation)
self._pdf_page.savefig(fig)
# do not want to close PDF when running evaluate
if 'PdfPages' in self._ctx.meta and \
('closef' not in self._ctx.meta or self._ctx.meta['closef']):
self._pdf_page.close()
# common protected functions
def _label(self, base, idx):
if self._legends is not None and len(self._legends) > idx:
return self._legends[idx]
if self.n_systems > 1:
return base + (" %d" % (idx + 1))
return base
def _set_axis(self):
if self._axlim is not None:
mpl.axis(self._axlim)
[docs]class Roc(PlotBase):
''' Handles the plotting of ROC'''
def __init__(self, ctx, scores, evaluation, func_load):
super(Roc, self).__init__(ctx, scores, evaluation, func_load)
self._titles = self._titles or ['ROC dev.', 'ROC eval.']
self._x_label = self._x_label or 'FPR'
self._y_label = self._y_label or "1 - FNR"
self._semilogx = ctx.meta.get('semilogx', True)
best_legend = 'lower right' if self._semilogx else 'upper right'
self._legend_loc = self._legend_loc or best_legend
# custom defaults
if self._axlim is None:
self._axlim = [None, None, -0.05, 1.05]
[docs] def compute(self, idx, input_scores, input_names):
''' Plot ROC for dev and eval data using
:py:func:`bob.measure.plot.roc`'''
neg_list, pos_list, _ = utils.get_fta_list(input_scores)
dev_neg, dev_pos = neg_list[0], pos_list[0]
dev_file = input_names[0]
if self._eval:
eval_neg, eval_pos = neg_list[1], pos_list[1]
eval_file = input_names[1]
mpl.figure(1)
if self._eval:
LOGGER.info("ROC dev. curve using %s", dev_file)
plot.roc_for_far(
dev_neg, dev_pos,
far_values=plot.log_values(self._min_dig or -4),
CAR=self._semilogx,
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label('dev', idx)
)
if self._split:
mpl.figure(2)
linestyle = '--' if not self._split else self._linestyles[idx]
LOGGER.info("ROC eval. curve using %s", eval_file)
plot.roc_for_far(
eval_neg, eval_pos, linestyle=linestyle,
far_values=plot.log_values(self._min_dig or -4),
CAR=self._semilogx,
color=self._colors[idx],
label=self._label('eval.', idx)
)
if self._far_at is not None:
from .. import fprfnr
for line in self._far_at:
thres_line = far_threshold(dev_neg, dev_pos, line)
eval_fmr, eval_fnmr = fprfnr(
eval_neg, eval_pos, thres_line)
eval_fnmr = 1 - eval_fnmr
mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30)
self._eval_points[line].append((eval_fmr, eval_fnmr))
else:
LOGGER.info("ROC dev. curve using %s", dev_file)
plot.roc_for_far(
dev_neg, dev_pos,
far_values=plot.log_values(self._min_dig or -4),
CAR=self._semilogx,
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label('dev', idx)
)
[docs]class Det(PlotBase):
''' Handles the plotting of DET '''
def __init__(self, ctx, scores, evaluation, func_load):
super(Det, self).__init__(ctx, scores, evaluation, func_load)
self._titles = self._titles or ['DET dev.', 'DET eval.']
self._x_label = self._x_label or 'FPR (%)'
self._y_label = self._y_label or 'FNR (%)'
self._legend_loc = self._legend_loc or 'upper right'
if self._far_at is not None:
self._trans_far_val = [ppndf(float(k)) for k in self._far_at]
# custom defaults here
if self._x_rotation is None:
self._x_rotation = 50
if self._axlim is None:
self._axlim = [0.01, 99, 0.01, 99]
if self._min_dig is not None:
self._axlim[0] = math.pow(10, self._min_dig) * 100
[docs] def compute(self, idx, input_scores, input_names):
''' Plot DET for dev and eval data using
:py:func:`bob.measure.plot.det`'''
neg_list, pos_list, _ = utils.get_fta_list(input_scores)
dev_neg, dev_pos = neg_list[0], pos_list[0]
dev_file = input_names[0]
if self._eval:
eval_neg, eval_pos = neg_list[1], pos_list[1]
eval_file = input_names[1]
mpl.figure(1)
if self._eval and eval_neg is not None:
LOGGER.info("DET dev. curve using %s", dev_file)
plot.det(
dev_neg, dev_pos, self._points, color=self._colors[idx],
linestyle=self._linestyles[idx],
label=self._label('dev.', idx)
)
if self._split:
mpl.figure(2)
linestyle = '--' if not self._split else self._linestyles[idx]
LOGGER.info("DET eval. curve using %s", eval_file)
plot.det(
eval_neg, eval_pos, self._points, color=self._colors[idx],
linestyle=linestyle,
label=self._label('eval.', idx)
)
if self._far_at is not None:
from .. import farfrr
for line in self._far_at:
thres_line = far_threshold(dev_neg, dev_pos, line)
eval_fmr, eval_fnmr = farfrr(
eval_neg, eval_pos, thres_line)
eval_fmr, eval_fnmr = ppndf(eval_fmr), ppndf(eval_fnmr)
mpl.scatter(eval_fmr, eval_fnmr, c=self._colors[idx], s=30)
self._eval_points[line].append((eval_fmr, eval_fnmr))
else:
LOGGER.info("DET dev. curve using %s", dev_file)
plot.det(
dev_neg, dev_pos, self._points, color=self._colors[idx],
linestyle=self._linestyles[idx],
label=self._label('dev.', idx)
)
def _set_axis(self):
plot.det_axis(self._axlim)
[docs]class Epc(PlotBase):
''' Handles the plotting of EPC '''
def __init__(self, ctx, scores, evaluation, func_load, hter='HTER'):
super(Epc, self).__init__(ctx, scores, evaluation, func_load)
if self._min_arg != 2:
raise click.UsageError("EPC requires dev. and eval. score files")
self._titles = self._titles or ['EPC'] * 2
self._x_label = self._x_label or r'$\alpha$'
self._y_label = self._y_label or hter + ' (%)'
self._legend_loc = self._legend_loc or 'upper center'
self._eval = True # always eval data with EPC
self._split = False
self._nb_figs = 1
self._far_at = None
[docs] def compute(self, idx, input_scores, input_names):
''' Plot EPC using :py:func:`bob.measure.plot.epc` '''
neg_list, pos_list, _ = utils.get_fta_list(input_scores)
dev_neg, dev_pos = neg_list[0], pos_list[0]
dev_file = input_names[0]
if self._eval:
eval_neg, eval_pos = neg_list[1], pos_list[1]
eval_file = input_names[1]
LOGGER.info("EPC using %s", dev_file + "_" + eval_file)
plot.epc(
dev_neg, dev_pos, eval_neg, eval_pos, self._points,
color=self._colors[idx], linestyle=self._linestyles[idx],
label=self._label(
'curve', idx
)
)
[docs]class Hist(PlotBase):
''' Functional base class for histograms'''
def __init__(self, ctx, scores, evaluation, func_load, nhist_per_system=2):
super(Hist, self).__init__(ctx, scores, evaluation, func_load)
self._nbins = ctx.meta.get('n_bins', ['doane'])
self._nhist_per_system = nhist_per_system
self._nbins = check_list_value(
self._nbins, nhist_per_system, 'n_bins',
'histograms')
self._thres = ctx.meta.get('thres')
self._thres = check_list_value(
self._thres, self.n_systems, 'thresholds')
self._criterion = ctx.meta.get('criterion')
# no vertical (threshold) is displayed
self._no_line = ctx.meta.get('no_line', False)
# subplot grid
self._nrows = ctx.meta.get('n_row', 1)
self._ncols = ctx.meta.get('n_col', 1)
# do not display dev histo
self._hide_dev = ctx.meta.get('hide_dev', False)
if self._hide_dev and not self._eval:
raise click.BadParameter(
"You can only use --hide-dev along with --eval")
# dev hist are displayed next to eval hist
self._nrows *= 1 if self._hide_dev or not self._eval else 2
self._nlegends = ctx.meta.get('legends_ncol', 3)
self._legend_loc = self._legend_loc or 'upper center'
# number of subplot on one page
self._step_print = int(self._nrows * self._ncols)
self._title_base = 'Scores'
self._y_label = self._y_label or 'Probability density'
self._x_label = self._x_label or 'Score values'
self._end_setup_plot = False
# overide _titles of PlotBase
self._titles = ctx.meta.get('titles', []) * 2
[docs] def compute(self, idx, input_scores, input_names):
''' Draw histograms of negative and positive scores.'''
dev_neg, dev_pos, eval_neg, eval_pos, threshold = \
self._get_neg_pos_thres(idx, input_scores, input_names)
# keep id of the current system
sys = idx
# if the id of the current system does not match the id of the plot,
# change it
if not self._hide_dev and self._eval:
row = int(idx / self._ncols) * 2
col = idx % self._ncols
idx = col + self._ncols * row
if not self._hide_dev or not self._eval:
self._print_subplot(idx, sys, dev_neg, dev_pos, threshold,
not self._no_line, False)
if self._eval:
idx += self._ncols if not self._hide_dev else 0
self._print_subplot(idx, sys, eval_neg, eval_pos, threshold,
not self._no_line, True)
def _print_subplot(self, idx, sys, neg, pos, threshold, draw_line, evaluation):
''' print a subplot for the given score and subplot index'''
n = idx % self._step_print
col = n % self._ncols
sub_plot_idx = n + 1
axis = mpl.subplot(self._nrows, self._ncols, sub_plot_idx)
self._setup_hist(neg, pos)
if col == 0:
axis.set_ylabel(self._y_label)
# systems per page
sys_per_page = self._step_print / (1 if self._hide_dev or not
self._eval else 2)
# rest to be printed
sys_idx = sys % sys_per_page
rest_print = self.n_systems - int(sys / sys_per_page) * sys_per_page
# lower histo only
is_lower = evaluation or not self._eval
if is_lower and sys_idx + self._ncols >= min(sys_per_page, rest_print):
axis.set_xlabel(self._x_label)
dflt_title = "Eval. scores" if evaluation else "Dev. scores"
if self.n_systems == 1 and (not self._eval or self._hide_dev):
dflt_title = " "
add = self.n_systems if is_lower else 0
axis.set_title(self._get_title(sys + add, dflt_title))
label = "%s threshold%s" % (
'' if self._criterion is None else
self._criterion.upper(), ' (dev)' if self._eval else ''
)
if draw_line:
self._lines(threshold, label, neg, pos, idx)
# enable the grid and set it below other elements
axis.set_axisbelow(True)
axis.grid(True, color=self._grid_color)
# if it was the last subplot of the page or the last subplot
# to display, save figure
if self._step_print == sub_plot_idx or (is_lower and sys ==
self.n_systems - 1):
# print legend on the page
self.plot_legends()
mpl.tight_layout()
self._pdf_page.savefig(mpl.gcf(), bbox_inches='tight')
mpl.clf()
mpl.figure()
def _get_title(self, idx, dflt=None):
''' Get the histo title for the given idx'''
title = self._titles[idx] if self._titles is not None \
and idx < len(self._titles) else dflt
title = title or self._title_base
title = '' if title is not None and not title.replace(
' ', '') else title
return title or ''
[docs] def plot_legends(self):
''' Print legend on current page'''
lines = []
labels = []
for ax in mpl.gcf().get_axes():
ali, ala = ax.get_legend_handles_labels()
# avoid duplicates in legend
for li, la in zip(ali, ala):
if la not in labels:
lines.append(li)
labels.append(la)
if self._disp_legend:
mpl.gcf().legend(
lines, labels, loc=self._legend_loc, fancybox=True,
framealpha=0.5, ncol=self._nlegends,
bbox_to_anchor=(0.55, 1.1),
)
def _get_neg_pos_thres(self, idx, input_scores, input_names):
''' Get scores and threshod for the given system at index idx'''
neg_list, pos_list, _ = utils.get_fta_list(input_scores)
length = len(neg_list)
# lists returned by get_fta_list contains all the following items:
# for bio or measure without eval:
# [dev]
# for vuln with {licit,spoof} with eval:
# [dev, eval]
# for vuln with {licit,spoof} without eval:
# [licit_dev, spoof_dev]
# for vuln with {licit,spoof} with eval:
# [licit_dev, licit_eval, spoof_dev, spoof_eval]
step = 2 if self._eval else 1
# can have several files for one system
dev_neg = [neg_list[x] for x in range(0, length, step)]
dev_pos = [pos_list[x] for x in range(0, length, step)]
eval_neg = eval_pos = None
if self._eval:
eval_neg = [neg_list[x] for x in range(1, length, step)]
eval_pos = [pos_list[x] for x in range(1, length, step)]
threshold = utils.get_thres(
self._criterion, dev_neg[0], dev_pos[0]
) if self._thres is None else self._thres[idx]
return dev_neg, dev_pos, eval_neg, eval_pos, threshold
def _density_hist(self, scores, n, **kwargs):
''' Plots one density histo'''
n, bins, patches = mpl.hist(
scores, density=True,
bins=self._nbins[n],
**kwargs
)
return (n, bins, patches)
def _lines(self, threshold, label=None, neg=None, pos=None,
idx=None, **kwargs):
''' Plots vertical line at threshold '''
label = label or 'Threshold'
kwargs.setdefault('color', 'C3')
kwargs.setdefault('linestyle', '--')
kwargs.setdefault('label', label)
# plot a vertical threshold line
mpl.axvline(x=threshold, ymin=0, ymax=1, **kwargs)
def _setup_hist(self, neg, pos):
''' This function can be overwritten in derived classes
Plots all the density histo required in one plot. Here negative and
positive scores densities.
'''
self._density_hist(
neg[0], n=0,
label='Negatives', alpha=0.5, color='C3'
)
self._density_hist(
pos[0], n=1,
label='Positives', alpha=0.5, color='C0'
)