Coverage for src/bob/pad/base/script/pad_commands.py: 94%
98 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-06 21:56 +0100
1"""The main entry for bob pad commands.
2"""
3import logging
4import os
6from csv import DictWriter
7from functools import partial
9import click
10import numpy
12from clapper.click import verbosity_option
14import bob.measure.script.figure as measure_figure
16from bob.measure.script import common_options
18from ..error_utils import split_csv_pad, split_csv_pad_per_pai
19from . import pad_figure as figure
21logger = logging.getLogger(__name__)
23SCORE_FORMAT = "Files must be in CSV format."
24CRITERIA = (
25 "eer",
26 "min-hter",
27 "far",
28 "bpcer5000",
29 "bpcer2000",
30 "bpcer1000",
31 "bpcer500",
32 "bpcer200",
33 "bpcer100",
34 "bpcer50",
35 "bpcer20",
36 "bpcer10",
37 "bpcer5",
38 "bpcer2",
39 "bpcer1",
40 "apcer5000",
41 "apcer2000",
42 "apcer1000",
43 "apcer500",
44 "apcer200",
45 "apcer100",
46 "apcer50",
47 "apcer20",
48 "apcer10",
49 "apcer5",
50 "apcer2",
51 "apcer1",
52)
55def metrics_option(
56 sname="-m",
57 lname="--metrics",
58 name="metrics",
59 help="List of metrics to print. Provide a string with comma separated metric "
60 "names. For possible values see the default value.",
61 default="apcer_pais,apcer_ap,bpcer,acer,fta,fpr,fnr,hter,far,frr,precision,recall,f1_score,auc,auc-log-scale",
62 **kwargs,
63):
64 """The metrics option"""
66 def custom_metrics_option(func):
67 def callback(ctx, param, value):
68 if value is not None:
69 value = value.split(",")
70 ctx.meta[name] = value
71 return value
73 return click.option(
74 sname,
75 lname,
76 default=default,
77 help=help,
78 show_default=True,
79 callback=callback,
80 **kwargs,
81 )(func)
83 return custom_metrics_option
86def regexps_option(
87 help="A list of regular expressions (by repeating this option) to be used to "
88 "categorize PAIs. Each regexp must match one type of PAI.",
89 **kwargs,
90):
91 def custom_regexps_option(func):
92 def callback(ctx, param, value):
93 ctx.meta["regexps"] = value
94 return value
96 return click.option(
97 "-r",
98 "--regexps",
99 default=None,
100 multiple=True,
101 help=help,
102 callback=callback,
103 **kwargs,
104 )(func)
106 return custom_regexps_option
109def regexp_column_option(
110 help="The column in the score files to match the regular expressions against.",
111 **kwargs,
112):
113 def custom_regexp_column_option(func):
114 def callback(ctx, param, value):
115 ctx.meta["regexp_column"] = value
116 return value
118 return click.option(
119 "-rc",
120 "--regexp-column",
121 default="attack_type",
122 help=help,
123 show_default=True,
124 callback=callback,
125 **kwargs,
126 )(func)
128 return custom_regexp_column_option
131def gen_pad_csv_scores(
132 filename, mean_match, mean_attacks, n_attack_types, n_clients, n_samples
133):
134 """Generates a CSV file containing random scores for PAD."""
135 columns = [
136 "claimed_id",
137 "test_label",
138 "is_bonafide",
139 "attack_type",
140 "sample_n",
141 "score",
142 ]
143 with open(filename, "w") as f:
144 writer = DictWriter(f, fieldnames=columns)
145 writer.writeheader()
146 # Bonafide rows
147 for client_id in range(n_clients):
148 for sample in range(n_samples):
149 writer.writerow(
150 {
151 "claimed_id": client_id,
152 "test_label": f"client/real/{client_id:03d}",
153 "is_bonafide": "True",
154 "attack_type": None,
155 "sample_n": sample,
156 "score": numpy.random.normal(loc=mean_match),
157 }
158 )
159 # Attacks rows
160 for attack_type in range(n_attack_types):
161 for client_id in range(n_clients):
162 for sample in range(n_samples):
163 writer.writerow(
164 {
165 "claimed_id": client_id,
166 "test_label": f"client/attack/{client_id:03d}",
167 "is_bonafide": "False",
168 "attack_type": f"type_{attack_type}",
169 "sample_n": sample,
170 "score": numpy.random.normal(
171 loc=mean_attacks[
172 attack_type % len(mean_attacks)
173 ]
174 ),
175 }
176 )
179@click.command()
180@click.argument("outdir")
181@click.option(
182 "-mm", "--mean-match", default=10, type=click.FLOAT, show_default=True
183)
184@click.option(
185 "-ma",
186 "--mean-attacks",
187 default=[-10, -6],
188 type=click.FLOAT,
189 show_default=True,
190 multiple=True,
191)
192@click.option(
193 "-c", "--n-clients", default=10, type=click.INT, show_default=True
194)
195@click.option("-s", "--n-samples", default=2, type=click.INT, show_default=True)
196@click.option("-a", "--n-attacks", default=2, type=click.INT, show_default=True)
197@verbosity_option(logger)
198@click.pass_context
199def gen(
200 ctx,
201 outdir,
202 mean_match,
203 mean_attacks,
204 n_clients,
205 n_samples,
206 n_attacks,
207 **kwargs,
208):
209 """Generate random scores.
210 Generates random scores in CSV format. The scores are generated
211 using Gaussian distribution whose mean is an input
212 parameter. The generated scores can be used as hypothetical datasets.
213 n-attacks defines the number of different type of attacks generated (like print and
214 mask). When multiples attacks are present, the mean-attacks option can be set
215 multiple times, specifying the mean of each attack scores distribution.
217 Example:
219 bob pad gen results/generated/scores-dev.csv -a 3 -ma 2 -ma 5 -ma 7 -mm 8
220 """
221 numpy.random.seed(0)
222 gen_pad_csv_scores(
223 os.path.join(outdir, "scores-dev.csv"),
224 mean_match,
225 mean_attacks,
226 n_attacks,
227 n_clients,
228 n_samples,
229 )
230 gen_pad_csv_scores(
231 os.path.join(outdir, "scores-eval.csv"),
232 mean_match,
233 mean_attacks,
234 n_attacks,
235 n_clients,
236 n_samples,
237 )
240@common_options.metrics_command(
241 common_options.METRICS_HELP.format(
242 names="FtA, APCER_AP, BPCER, FPR, FNR, FAR, FRR, ACER, HTER, precision, recall, f1_score",
243 criteria=CRITERIA,
244 score_format=SCORE_FORMAT,
245 hter_note="Note that APCER_AP = max(APCER_pais), BPCER=FNR, "
246 "FAR = FPR * (1 - FtA), "
247 "FRR = FtA + FNR * (1 - FtA), "
248 "ACER = (APCER_AP + BPCER) / 2, "
249 "and HTER = (FPR + FNR) / 2. "
250 "You can control which metrics are printed using the --metrics option. "
251 "You can use --regexps and --regexp_column options to change the behavior "
252 "of finding Presentation Attack Instrument (PAI) types",
253 command="bob pad metrics",
254 ),
255 criteria=CRITERIA,
256 check_criteria=False,
257 epilog="""\b
258More Examples:
259\b
260bob pad metrics -vvv -e -lg IQM,LBP -r print -r video -m fta,apcer_pais,apcer_ap,bpcer,acer,hter \
261/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_1/scores/scores-{dev,eval}
263See also ``bob pad multi-metrics``.
264""",
265)
266@regexps_option()
267@regexp_column_option()
268@metrics_option()
269def metrics(ctx, scores, evaluation, regexps, regexp_column, metrics, **kwargs):
270 load_fn = partial(
271 split_csv_pad_per_pai, regexps=regexps, regexp_column=regexp_column
272 )
273 process = figure.Metrics(ctx, scores, evaluation, load_fn, metrics)
274 process.run()
277@common_options.roc_command(
278 common_options.ROC_HELP.format(
279 score_format=SCORE_FORMAT, command="bob pad roc"
280 )
281)
282def roc(ctx, scores, evaluation, **kwargs):
283 process = figure.Roc(ctx, scores, evaluation, split_csv_pad)
284 process.run()
287@common_options.det_command(
288 common_options.DET_HELP.format(
289 score_format=SCORE_FORMAT, command="bob pad det"
290 )
291)
292def det(ctx, scores, evaluation, **kwargs):
293 process = figure.Det(ctx, scores, evaluation, split_csv_pad)
294 process.run()
297@common_options.epc_command(
298 common_options.EPC_HELP.format(
299 score_format=SCORE_FORMAT, command="bob pad epc"
300 )
301)
302def epc(ctx, scores, **kwargs):
303 process = measure_figure.Epc(ctx, scores, True, split_csv_pad, hter="ACER")
304 process.run()
307@common_options.hist_command(
308 common_options.HIST_HELP.format(
309 score_format=SCORE_FORMAT, command="bob pad hist"
310 )
311)
312def hist(ctx, scores, evaluation, **kwargs):
313 process = figure.Hist(ctx, scores, evaluation, split_csv_pad)
314 process.run()
317@common_options.evaluate_command(
318 common_options.EVALUATE_HELP.format(
319 score_format=SCORE_FORMAT, command="bob pad evaluate"
320 ),
321 criteria=CRITERIA,
322)
323def evaluate(ctx, scores, evaluation, **kwargs):
324 common_options.evaluate_flow(
325 ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs
326 )
329@common_options.multi_metrics_command(
330 common_options.MULTI_METRICS_HELP.format(
331 names="FtA, APCER, BPCER, FAR, FRR, ACER, HTER, precision, recall, f1_score",
332 criteria=CRITERIA,
333 score_format=SCORE_FORMAT,
334 command="bob pad multi-metrics",
335 ),
336 criteria=CRITERIA,
337 epilog="""\b
338More examples:
340\b
341bob pad multi-metrics -vvv -e -pn 6 -lg IQM,LBP -r print -r video \
342/scores/oulunpu/{qm-svm,lbp-svm}/Protocol_3_{1,2,3,4,5,6}/scores/scores-{dev,eval}
344See also ``bob pad metrics``.
345""",
346)
347@regexps_option()
348@regexp_column_option()
349@metrics_option(default="fta,apcer_pais,apcer_ap,bpcer,acer,hter")
350def multi_metrics(
351 ctx,
352 scores,
353 evaluation,
354 protocols_number,
355 regexps,
356 regexp_column,
357 metrics,
358 **kwargs,
359):
360 ctx.meta["min_arg"] = protocols_number * (2 if evaluation else 1)
361 load_fn = partial(
362 split_csv_pad_per_pai, regexps=regexps, regexp_column=regexp_column
363 )
364 process = figure.MultiMetrics(ctx, scores, evaluation, load_fn, metrics)
365 process.run()