Coverage for src/bob/bio/base/script/figure.py: 88%
145 statements
« prev ^ index » next coverage.py v7.6.0, created at 2024-07-12 22:34 +0200
1"""Plots and measures for bob.bio.base"""
3import logging
4import math
6import click
7import matplotlib.pyplot as mpl
9from tabulate import tabulate
11import bob.measure
12import bob.measure.script.figure as measure_figure
14from bob.measure import plot
16LOGGER = logging.getLogger("bob.bio.base")
19class Roc(measure_figure.Roc):
20 def __init__(self, ctx, scores, evaluation, func_load):
21 super(Roc, self).__init__(ctx, scores, evaluation, func_load)
22 self._x_label = ctx.meta.get("x_label") or "FMR"
23 default_y_label = "1 - FNMR" if self._tpr else "FNMR"
24 self._y_label = ctx.meta.get("y_label") or default_y_label
27class Det(measure_figure.Det):
28 def __init__(self, ctx, scores, evaluation, func_load):
29 super(Det, self).__init__(ctx, scores, evaluation, func_load)
30 self._x_label = ctx.meta.get("x_label") or "FMR (%)"
31 self._y_label = ctx.meta.get("y_label") or "FNMR (%)"
34class Cmc(measure_figure.PlotBase):
35 """Handles the plotting of Cmc"""
37 def __init__(self, ctx, scores, evaluation, func_load):
38 super(Cmc, self).__init__(ctx, scores, evaluation, func_load)
39 self._semilogx = ctx.meta.get("semilogx", True)
40 self._titles = self._titles or ["CMC dev.", "CMC eval."]
41 self._x_label = self._x_label or "Rank"
42 self._y_label = self._y_label or "Identification rate"
43 self._max_R = 0
45 def compute(self, idx, input_scores, input_names):
46 """Plot CMC for dev and eval data using
47 :py:func:`bob.measure.plot.cmc`"""
48 mpl.figure(1)
49 if self._eval:
50 linestyle = "-" if not self._split else self._linestyles[idx]
51 LOGGER.info("CMC dev. curve using %s", input_names[0])
52 rank = plot.cmc(
53 input_scores[0],
54 logx=self._semilogx,
55 color=self._colors[idx],
56 linestyle=linestyle,
57 label=self._label("dev.", idx),
58 )
59 self._max_R = max(rank, self._max_R)
60 linestyle = "--"
61 if self._split:
62 mpl.figure(2)
63 linestyle = self._linestyles[idx]
65 LOGGER.info("CMC eval. curve using %s", input_names[1])
66 rank = plot.cmc(
67 input_scores[1],
68 logx=self._semilogx,
69 color=self._colors[idx],
70 linestyle=linestyle,
71 label=self._label("eval.", idx),
72 )
73 self._max_R = max(rank, self._max_R)
74 else:
75 LOGGER.info("CMC dev. curve using %s", input_names[0])
76 rank = plot.cmc(
77 input_scores[0],
78 logx=self._semilogx,
79 color=self._colors[idx],
80 linestyle=self._linestyles[idx],
81 label=self._label("dev.", idx),
82 )
83 self._max_R = max(rank, self._max_R)
86class Dir(measure_figure.PlotBase):
87 """Handles the plotting of DIR curve"""
89 def __init__(self, ctx, scores, evaluation, func_load):
90 super(Dir, self).__init__(ctx, scores, evaluation, func_load)
91 self._semilogx = ctx.meta.get("semilogx", True)
92 self._rank = ctx.meta.get("rank", 1)
93 self._titles = self._titles or ["DIR curve"] * 2
94 self._x_label = self._x_label or "False Positive Identification Rate"
95 self._y_label = self._y_label or "True Positive Identification Rate"
97 def compute(self, idx, input_scores, input_names):
98 """Plot DIR for dev and eval data using
99 :py:func:`bob.measure.plot.detection_identification_curve`"""
100 mpl.figure(1)
101 if self._eval:
102 linestyle = "-" if not self._split else self._linestyles[idx]
103 LOGGER.info("DIR dev. curve using %s", input_names[0])
104 plot.detection_identification_curve(
105 input_scores[0],
106 rank=self._rank,
107 logx=self._semilogx,
108 color=self._colors[idx],
109 linestyle=linestyle,
110 label=self._label("dev", idx),
111 )
112 linestyle = "--"
113 if self._split:
114 mpl.figure(2)
115 linestyle = self._linestyles[idx]
117 LOGGER.info("DIR eval. curve using %s", input_names[1])
118 plot.detection_identification_curve(
119 input_scores[1],
120 rank=self._rank,
121 logx=self._semilogx,
122 color=self._colors[idx],
123 linestyle=linestyle,
124 label=self._label("eval", idx),
125 )
126 else:
127 LOGGER.info("DIR dev. curve using %s", input_names[0])
128 plot.detection_identification_curve(
129 input_scores[0],
130 rank=self._rank,
131 logx=self._semilogx,
132 color=self._colors[idx],
133 linestyle=self._linestyles[idx],
134 label=self._label("dev", idx),
135 )
137 if self._min_dig is not None:
138 mpl.xlim(xmin=math.pow(10, self._min_dig))
141class Metrics(measure_figure.Metrics):
142 """Compute metrics from score files"""
144 def __init__(
145 self,
146 ctx,
147 scores,
148 evaluation,
149 func_load,
150 names=(
151 "Failure to Acquire",
152 "False Match Rate",
153 "False Non Match Rate",
154 "False Accept Rate",
155 "False Reject Rate",
156 "Half Total Error Rate",
157 ),
158 ):
159 super(Metrics, self).__init__(ctx, scores, evaluation, func_load, names)
161 def init_process(self):
162 if self._criterion == "rr":
163 self._thres = (
164 [None] * self.n_systems if self._thres is None else self._thres
165 )
167 def compute(self, idx, input_scores, input_names):
168 """Compute metrics for the given criteria"""
169 title = self._legends[idx] if self._legends is not None else None
170 headers = ["" or title, "Dev. %s" % input_names[0]]
171 if self._eval and input_scores[1] is not None:
172 headers.append("eval % s" % input_names[1])
173 if self._criterion == "rr":
174 rr = bob.measure.recognition_rate(input_scores[0], self._thres[idx])
175 dev_rr = "%.1f%%" % (100 * rr)
176 raws = [["RR", dev_rr]]
177 if self._eval and input_scores[1] is not None:
178 rr = bob.measure.recognition_rate(
179 input_scores[1], self._thres[idx]
180 )
181 eval_rr = "%.1f%%" % (100 * rr)
182 raws[0].append(eval_rr)
183 click.echo(
184 tabulate(raws, headers, self._tablefmt), file=self.log_file
185 )
186 elif self._criterion == "mindcf":
187 if "cost" in self._ctx.meta:
188 cost = self._ctx.meta.get("cost", 0.99)
189 threshold = (
190 bob.measure.min_weighted_error_rate_threshold(
191 input_scores[0][0], input_scores[0][1], cost
192 )
193 if self._thres is None
194 else self._thres[idx]
195 )
196 if self._thres is None:
197 click.echo(
198 "[minDCF - Cost:%f] Threshold on Development set `%s`: %e"
199 % (cost, input_names[0], threshold),
200 file=self.log_file,
201 )
202 else:
203 click.echo(
204 "[minDCF] User defined Threshold: %e" % threshold,
205 file=self.log_file,
206 )
207 # apply threshold to development set
208 far, frr = bob.measure.farfrr(
209 input_scores[0][0], input_scores[0][1], threshold
210 )
211 dev_far_str = "%.1f%%" % (100 * far)
212 dev_frr_str = "%.1f%%" % (100 * frr)
213 dev_mindcf_str = "%.1f%%" % (
214 (cost * far + (1 - cost) * frr) * 100.0
215 )
216 raws = [
217 ["FAR", dev_far_str],
218 ["FRR", dev_frr_str],
219 ["minDCF", dev_mindcf_str],
220 ]
221 if self._eval and input_scores[1] is not None:
222 # apply threshold to development set
223 far, frr = bob.measure.farfrr(
224 input_scores[1][0], input_scores[1][1], threshold
225 )
226 eval_far_str = "%.1f%%" % (100 * far)
227 eval_frr_str = "%.1f%%" % (100 * frr)
228 eval_mindcf_str = "%.1f%%" % (
229 (cost * far + (1 - cost) * frr) * 100.0
230 )
231 raws[0].append(eval_far_str)
232 raws[1].append(eval_frr_str)
233 raws[2].append(eval_mindcf_str)
234 click.echo(
235 tabulate(raws, headers, self._tablefmt), file=self.log_file
236 )
237 elif self._criterion == "cllr":
238 cllr = bob.measure.calibration.cllr(
239 input_scores[0][0], input_scores[0][1]
240 )
241 min_cllr = bob.measure.calibration.min_cllr(
242 input_scores[0][0], input_scores[0][1]
243 )
244 dev_cllr_str = "%.1f%%" % cllr
245 dev_min_cllr_str = "%.1f%%" % min_cllr
246 raws = [["Cllr", dev_cllr_str], ["minCllr", dev_min_cllr_str]]
247 if self._eval and input_scores[1] is not None:
248 cllr = bob.measure.calibration.cllr(
249 input_scores[1][0], input_scores[1][1]
250 )
251 min_cllr = bob.measure.calibration.min_cllr(
252 input_scores[1][0], input_scores[1][1]
253 )
254 eval_cllr_str = "%.1f%%" % cllr
255 eval_min_cllr_str = "%.1f%%" % min_cllr
256 raws[0].append(eval_cllr_str)
257 raws[1].append(eval_min_cllr_str)
258 click.echo(
259 tabulate(raws, headers, self._tablefmt), file=self.log_file
260 )
261 else:
262 title = self._legends[idx] if self._legends is not None else None
263 all_metrics = self._get_all_metrics(idx, input_scores, input_names)
264 headers = [" " or title, "Development"]
265 rows = [
266 [self.names[0], all_metrics[0][0]],
267 [self.names[1], all_metrics[0][1]],
268 [self.names[2], all_metrics[0][2]],
269 [self.names[3], all_metrics[0][3]],
270 [self.names[4], all_metrics[0][4]],
271 [self.names[5], all_metrics[0][5]],
272 ]
274 if self._eval:
275 # computes statistics for the eval set based on the threshold a
276 # priori
277 headers.append("Evaluation")
278 rows[0].append(all_metrics[1][0])
279 rows[1].append(all_metrics[1][1])
280 rows[2].append(all_metrics[1][2])
281 rows[3].append(all_metrics[1][3])
282 rows[4].append(all_metrics[1][4])
283 rows[5].append(all_metrics[1][5])
285 click.echo(
286 tabulate(rows, headers, self._tablefmt), file=self.log_file
287 )
290class MultiMetrics(measure_figure.MultiMetrics):
291 """Compute metrics from score files"""
293 def __init__(self, ctx, scores, evaluation, func_load):
294 super(MultiMetrics, self).__init__(
295 ctx,
296 scores,
297 evaluation,
298 func_load,
299 names=(
300 "Failure to Acquire",
301 "False Match Rate",
302 "False Non Match Rate",
303 "False Accept Rate",
304 "False Reject Rate",
305 "Half Total Error Rate",
306 ),
307 )
310class Hist(measure_figure.Hist):
311 """Histograms for biometric scores"""
313 def _setup_hist(self, neg, pos):
314 self._title_base = "Biometric scores"
315 self._density_hist(pos[0], n=0, label="Genuines", alpha=0.9, color="C2")
316 self._density_hist(
317 neg[0], n=1, label="Zero-effort impostors", alpha=0.8, color="C0"
318 )