Coverage for src/bob/bio/base/script/vuln_commands.py: 87%
173 statements
« prev ^ index » next coverage.py v7.6.0, created at 2024-07-12 22:34 +0200
1"""The click-based vulnerability analysis commands.
2"""
4import csv
5import functools
6import logging
7import os
9import click
11from clapper.click import verbosity_option
12from click.types import FLOAT
13from numpy import random
15from bob.bio.base.score.load import split_csv_vuln
16from bob.io.base import create_directories_safe
17from bob.measure.script import common_options
19from . import vuln_figure as figure
21logger = logging.getLogger(__name__)
24def vuln_plot_options(
25 docstring,
26 plot_output_default="vuln_plot.pdf",
27 legend_loc_default="best",
28 axes_lim_default=None,
29 figsize_default="4,3",
30 force_eval=False,
31 x_label_rotation_default=0,
32):
33 def custom_options_command(func):
34 func.__doc__ = docstring
36 def eval_if_not_forced(force_eval):
37 def decorator(f):
38 if not force_eval:
39 return common_options.eval_option()(
40 common_options.sep_dev_eval_option()(
41 common_options.hide_dev_option()(f)
42 )
43 )
44 else:
45 return f
47 return decorator
49 @click.command()
50 @common_options.scores_argument(
51 min_arg=1, force_eval=force_eval, nargs=-1
52 )
53 @eval_if_not_forced(force_eval)
54 @common_options.legends_option()
55 @common_options.no_legend_option()
56 @common_options.legend_ncols_option()
57 @common_options.legend_loc_option(dflt=legend_loc_default)
58 @common_options.output_plot_file_option(default_out=plot_output_default)
59 @common_options.lines_at_option(dflt=" ")
60 @common_options.axes_val_option(dflt=axes_lim_default)
61 @common_options.x_rotation_option(dflt=x_label_rotation_default)
62 @common_options.x_label_option()
63 @common_options.y_label_option()
64 @common_options.points_curve_option()
65 @common_options.const_layout_option()
66 @common_options.figsize_option(dflt=figsize_default)
67 @common_options.style_option()
68 @common_options.linestyles_option()
69 @common_options.alpha_option()
70 @verbosity_option(logger=logger)
71 @click.pass_context
72 @functools.wraps(func)
73 def wrapper(*args, **kwds):
74 return func(*args, **kwds)
76 return wrapper
78 return custom_options_command
81def real_data_option(**kwargs):
82 """Option to choose if input data is real or generated"""
83 return common_options.bool_option(
84 name="real-data",
85 short_name="R",
86 desc="If False, will annotate the plots hypothetically, instead "
87 "of with real data values of the calculated error rates.",
88 dflt=True,
89 **kwargs,
90 )
93def fnmr_at_option(dflt=" ", **kwargs):
94 """Get option to draw const FNMR lines"""
95 return common_options.list_float_option(
96 name="fnmr",
97 short_name="fnmr",
98 desc="If given, draw horizontal lines at the given FNMR position. "
99 "Your values must be separated with a comma (,) without space. "
100 "This option works in ROC and DET curves.",
101 nitems=None,
102 dflt=dflt,
103 **kwargs,
104 )
107def gen_score_distr(
108 mean_gen,
109 mean_zei,
110 mean_pa,
111 sigma_gen=1,
112 sigma_zei=1,
113 sigma_pa=1,
114 num_gen=5000,
115 num_zei=5000,
116 num_pa=5000,
117):
118 # initialise the random number generator
119 mt = random.RandomState(0)
121 genuine_scores = mt.normal(loc=mean_gen, scale=sigma_gen, size=num_gen)
122 zei_scores = mt.normal(loc=mean_zei, scale=sigma_zei, size=num_zei)
123 pa_scores = mt.normal(loc=mean_pa, scale=sigma_pa, size=num_pa)
125 return genuine_scores, zei_scores, pa_scores
128def write_scores_to_file(neg_licit, pos_licit, spoof, filename):
129 """Writes score distributions into a CSV score file. For the format of
130 the score files, please refer to Bob's documentation.
132 Parameters
133 ----------
134 neg : array_like
135 Scores for negative samples.
136 pos : array_like
137 Scores for positive samples.
138 filename : str
139 The path to write the score to.
140 """
141 logger.info(f"Creating score file '{filename}'")
142 create_directories_safe(os.path.dirname(filename))
143 with open(filename, "wt") as f:
144 csv_writer = csv.writer(f)
145 # Write the header
146 csv_writer.writerow(
147 [
148 "bio_ref_subject_id",
149 "probe_subject_id",
150 "probe_key",
151 "probe_attack_type",
152 "score",
153 ]
154 )
155 for score in neg_licit:
156 csv_writer.writerow(["x", "y", "0", None, score])
157 for score in pos_licit:
158 csv_writer.writerow(["x", "x", "0", None, score])
159 for score in spoof:
160 csv_writer.writerow(["x", "y", "0", "pai", score])
163@click.command()
164@click.argument("outdir")
165@click.option("-mg", "--mean-gen", default=7, type=FLOAT, show_default=True)
166@click.option("-mz", "--mean-zei", default=3, type=FLOAT, show_default=True)
167@click.option("-mp", "--mean-pa", default=5, type=FLOAT, show_default=True)
168@verbosity_option(logger=logger)
169def gen(outdir, mean_gen, mean_zei, mean_pa, **kwargs):
170 """Generate random scores.
171 Generates random scores for three types of verification attempts:
172 genuine users, zero-effort impostors and spoofing attacks and writes them
173 into CSV score files for so called licit and spoof scenario. The
174 scores are generated using Gaussian distribution whose mean is an input
175 parameter. The generated scores can be used as hypothetical datasets.
176 """
177 # Generate the data
178 genuine_dev, zei_dev, pa_dev = gen_score_distr(mean_gen, mean_zei, mean_pa)
179 genuine_eval, zei_eval, pa_eval = gen_score_distr(
180 mean_gen, mean_zei, mean_pa
181 )
183 # Write the data into files
184 write_scores_to_file(
185 zei_dev, genuine_dev, pa_dev, os.path.join(outdir, "scores-dev.csv")
186 )
187 write_scores_to_file(
188 zei_eval, genuine_eval, pa_eval, os.path.join(outdir, "scores-eval.csv")
189 )
192@common_options.metrics_command(
193 docstring="""Extracts different statistical values from scores distributions
195 Prints a table that contains different metrics to assess the performance of a
196 biometric system against zero-effort impostor and presentation attacks.
198 The CSV score files must contain an `attack-type` column, in addition to the
199 "regular" biometric scores columns (`bio_ref_subject_id`,
200 `probe_subject_id`, and `score`).
202 Examples:
204 $ bob vuln metrics -v scores-dev.csv
206 $ bob vuln metrics -v -e scores-{dev,eval}.csv
207 """
208)
209@common_options.cost_option()
210def metrics(ctx, scores, evaluation, **kwargs):
211 process = figure.Metrics(ctx, scores, evaluation, split_csv_vuln)
212 process.run()
215@vuln_plot_options(
216 docstring="""Plots the ROC for vulnerability analysis
218 You need to provide 1 or 2 (with `--eval`) score
219 files for each vulnerability system in this order:
221 \b
222 * dev scores
223 * [eval scores]
225 The CSV score files must contain an `attack-type` column, in addition to the
226 "regular" biometric scores columns (`bio_ref_subject_id`,
227 `probe_subject_id`, and `score`).
229 Examples:
231 $ bob vuln roc -v -o roc.pdf scores.csv
233 $ bob vuln roc -v -e scores-{dev,eval}.csv
234 """,
235 plot_output_default="vuln_roc.pdf",
236)
237@common_options.title_option()
238@common_options.min_far_option()
239@common_options.tpr_option(dflt=True)
240@common_options.semilogx_option(dflt=True)
241@fnmr_at_option()
242@real_data_option()
243def roc(ctx, scores, evaluation, real_data, **kwargs):
244 process = figure.RocVuln(
245 ctx, scores, evaluation, split_csv_vuln, real_data, False
246 )
247 process.run()
250@vuln_plot_options(
251 docstring="""Plots the DET for vulnerability analysis
253 You need to provide 1 or 2 (with `--eval`) score
254 files for each vulnerability system in this order:
256 \b
257 * dev scores
258 * [eval scores]
260 The CSV score files must contain an `attack-type` column, in addition to the
261 "regular" biometric scores columns (`bio_ref_subject_id`,
262 `probe_subject_id`, and `score`).
264 See :ref:`bob.bio.base.vulnerability` in the documentation for a guide on
265 vulnerability analysis.
267 Examples:
269 $ bob vuln det -v -o det.pdf scores.csv
271 $ bob vuln det -v -e scores-{dev,eval}.csv
272 """,
273 plot_output_default="vuln_det.pdf",
274 legend_loc_default="upper-right",
275 axes_lim_default="0.01,95,0.01,95",
276 figsize_default="6,4",
277 x_label_rotation_default=45,
278)
279@common_options.title_option()
280@real_data_option()
281@fnmr_at_option()
282def det(ctx, scores, evaluation, real_data, **kwargs):
283 process = figure.DetVuln(
284 ctx, scores, evaluation, split_csv_vuln, real_data, False
285 )
286 process.run()
289@vuln_plot_options(
290 docstring="""Plots the EPC for vulnerability analysis
292 You need to provide 2 score files for each vulnerability system in this order:
294 \b
295 * dev scores
296 * eval scores
298 The CSV score files must contain an `attack-type` column, in addition to the
299 "regular" biometric scores columns (`bio_ref_subject_id`,
300 `probe_subject_id`, and `score`).
302 See :ref:`bob.bio.base.vulnerability` in the documentation for a guide on
303 vulnerability analysis.
305 Examples:
307 $ bob vuln epc -v scores-dev.csv scores-eval.csv
309 $ bob vuln epc -v -o epc.pdf scores-{dev,eval}.csv
310 """,
311 plot_output_default="vuln_epc.pdf",
312 force_eval=True,
313 legend_loc_default="upper-center",
314)
315@common_options.title_option()
316@common_options.bool_option(
317 "iapmr", "I", "Whether to plot the IAPMR related lines or not.", True
318)
319def epc(ctx, scores, **kwargs):
320 process = figure.Epc(ctx, scores, True, split_csv_vuln)
321 process.run()
324@vuln_plot_options(
325 docstring="""Plots the EPSC for vulnerability analysis
327 Plots the Expected Performance Spoofing Curve.
329 Note that when using 3D plots with option ``--three-d``, you cannot plot
330 both WER and IAPMR on the same figure (which is possible in 2D).
332 You need to provide 2 score files for each vulnerability system in this order:
334 \b
335 * dev scores
336 * eval scores
338 The CSV score files must contain an `attack-type` column, in addition to the
339 "regular" biometric scores columns (`bio_ref_subject_id`,
340 `probe_subject_id`, and `score`).
342 See :ref:`bob.bio.base.vulnerability` in the documentation for a guide on
343 vulnerability analysis.
345 Examples:
347 $ bob vuln epsc -v scores-dev.csv scores-eval.csv
349 $ bob vuln epsc -v -o epsc.pdf scores-{dev,eval}.csv
351 $ bob vuln epsc -v -D -o epsc_3D.pdf scores-{dev,eval}.csv
352 """,
353 plot_output_default="vuln_epc.pdf",
354 force_eval=True,
355 figsize_default="5,3",
356)
357@common_options.titles_option()
358@common_options.bool_option(
359 "wer", "w", "Whether to plot the WER related lines or not.", True
360)
361@common_options.bool_option(
362 "iapmr", "I", "Whether to plot the IAPMR related lines or not.", True
363)
364@common_options.bool_option(
365 "three-d",
366 "D",
367 "If true, generate 3D plots. You need to turn off "
368 "wer or iapmr when using this option.",
369 False,
370)
371@click.option(
372 "-c",
373 "--criteria",
374 default="eer",
375 show_default=True,
376 help="Criteria for threshold selection",
377 type=click.Choice(("eer", "min-hter")),
378)
379@click.option(
380 "-vp",
381 "--var-param",
382 default="omega",
383 show_default=True,
384 help="Name of the varying parameter",
385 type=click.Choice(("omega", "beta")),
386)
387@common_options.list_float_option(
388 name="fixed-params",
389 short_name="fp",
390 dflt="0.5",
391 desc="Values of the fixed parameter, separated by commas",
392)
393@click.option(
394 "-s",
395 "--sampling",
396 default=5,
397 show_default=True,
398 help="Sampling of the EPSC 3D surface",
399 type=click.INT,
400)
401def epsc(ctx, scores, criteria, var_param, three_d, sampling, **kwargs):
402 if three_d:
403 if ctx.meta["wer"] and ctx.meta["iapmr"]:
404 logger.info(
405 "Cannot plot both WER and IAPMR in 3D. Will turn IAPMR off."
406 )
407 ctx.meta["iapmr"] = False
408 ctx.meta["sampling"] = sampling
409 process = figure.Epsc3D(
410 ctx, scores, split_csv_vuln, criteria, var_param, **kwargs
411 )
412 else:
413 process = figure.Epsc(
414 ctx, scores, split_csv_vuln, criteria, var_param, **kwargs
415 )
416 process.run()
419@vuln_plot_options(
420 docstring="""Vulnerability analysis score distribution histograms.
422 Plots the histogram of score distributions. You need to provide 1 or 2 score
423 files for each biometric system in this order:
425 \b
426 * development scores
427 * [evaluation scores]
429 When evaluation scores are provided, you must use the ``--eval`` option.
431 The CSV score files must contain an `attack-type` column, in addition to the
432 "regular" biometric scores columns (`bio_ref_subject_id`,
433 `probe_subject_id`, and `score`).
435 See :ref:`bob.bio.base.vulnerability` in the documentation for a guide on
436 vulnerability analysis.
438 When eval-scores are given, eval-scores histograms are displayed with the
439 threshold line computed from dev-scores.
441 Examples:
443 $ bob vuln hist -v -o hist.pdf results/scores-dev.csv
445 $ bob vuln hist -e -v results/scores-dev.csv results/scores-eval.csv
447 $ bob vuln hist -e -v results/scores-{dev,eval}.csv
448 """,
449 plot_output_default="vuln_hist.pdf",
450)
451@common_options.titles_option()
452@common_options.n_bins_option()
453@common_options.thresholds_option()
454@common_options.print_filenames_option(dflt=False)
455@common_options.bool_option(
456 "iapmr-line", "I", "Whether to plot the IAPMR related lines or not.", True
457)
458@real_data_option()
459@common_options.subplot_option()
460@common_options.criterion_option()
461def hist(ctx, scores, evaluation, **kwargs):
462 process = figure.HistVuln(ctx, scores, evaluation, split_csv_vuln)
463 process.run()
466@vuln_plot_options(
467 docstring="""Plots the FMR vs IAPMR for vulnerability analysis
469 You need to provide 1 or 2 (with `--eval`) score
470 files for each vulnerability system in this order:
472 \b
473 * dev scores
474 * [eval scores]
476 The CSV score files must contain an `attack-type` column, in addition to the
477 "regular" biometric scores columns (`bio_ref_subject_id`,
478 `probe_subject_id`, and `score`).
480 Examples:
482 $ bob vuln fmr_iapmr -v -o fmr_iapmr.pdf scores.csv
484 $ bob vuln fmr_iapmr -v -e scores-{dev,eval}.csv
485 """,
486 plot_output_default="vuln_roc.pdf",
487 force_eval=True,
488)
489@common_options.title_option()
490@common_options.semilogx_option()
491def fmr_iapmr(ctx, scores, **kwargs):
492 process = figure.FmrIapmr(ctx, scores, True, split_csv_vuln)
493 process.run()
496@common_options.evaluate_command(
497 common_options.EVALUATE_HELP.format(
498 score_format=(
499 "Files must be in CSV format, with the `bio_ref_subject_id`, "
500 "`probe_references_id`, `score`, and `attack_type` columns."
501 ),
502 command="bob vuln evaluate",
503 ),
504 criteria=("eer", "min-hter", "far"),
505)
506def evaluate(ctx, scores, evaluation, **kwargs):
507 # open_mode is always 'write' in this command.
508 ctx.meta["open_mode"] = "w"
509 criterion = ctx.meta.get("criterion")
510 if criterion is not None:
511 click.echo(f"Computing metrics with {criterion}...")
512 ctx.invoke(metrics, scores=scores, evaluation=evaluation)
513 if ctx.meta.get("log") is not None:
514 click.echo(f"[metrics] => {ctx.meta['log']}")
516 ctx.meta["lines_at"] = None
518 # Avoid closing pdf file before all figures are plotted
519 ctx.meta["closef"] = False
520 if evaluation:
521 click.echo("Starting evaluate with dev and eval scores...")
522 else:
523 click.echo("Starting evaluate with dev scores only...")
524 click.echo("Plotting FMR vs IAPMR for bob vuln evaluate...")
525 ctx.forward(fmr_iapmr) # uses class defaults plot settings
526 click.echo("Plotting ROC for bob vuln evaluate...")
527 ctx.forward(roc) # uses class defaults plot settings
528 click.echo("Plotting DET for bob vuln evaluate...")
529 ctx.forward(det) # uses class defaults plot settings
530 if evaluation:
531 click.echo("Plotting EPSC for bob vuln evaluate...")
532 ctx.forward(epsc) # uses class defaults plot settings
533 # Mark the last plot to close the output file
534 ctx.meta["closef"] = True
535 click.echo("Plotting score histograms for bob vuln evaluate...")
536 ctx.forward(hist)
537 click.echo("Evaluate successfully completed!")
538 click.echo(f"[plots] => {ctx.meta['output']}")