Coverage for src/bob/bio/base/script/commands.py: 92%
100 statements
« prev ^ index » next coverage.py v7.6.0, created at 2024-07-12 22:34 +0200
1""" Click commands for ``bob.bio.base`` """
2import functools
3import logging
5import click
7from clapper.click import verbosity_option
9import bob.measure.script.figure as measure_figure
11from bob.measure.script import common_options
13from ..score import load
14from . import figure as bio_figure
16logger = logging.getLogger(__name__)
18SCORE_FORMAT = (
19 "Files must be 4- or 5- columns format, see "
20 ":py:func:`bob.bio.base.score.load.four_column` and "
21 ":py:func:`bob.bio.base.score.load.five_column` for details."
22)
23CRITERIA = ("eer", "min-hter", "far", "mindcf", "cllr", "rr")
26def rank_option(**kwargs):
27 """Get option for rank parameter"""
29 def custom_rank_option(func):
30 def callback(ctx, param, value):
31 value = 1 if value < 0 else value
32 ctx.meta["rank"] = value
33 return value
35 return click.option(
36 "-rk",
37 "--rank",
38 type=click.INT,
39 default=1,
40 help="Provide rank for the command",
41 callback=callback,
42 show_default=True,
43 **kwargs,
44 )(func)
46 return custom_rank_option
49@common_options.metrics_command(
50 common_options.METRICS_HELP.format(
51 names="FtA, FAR, FRR, FMR, FNMR, HTER",
52 criteria=CRITERIA,
53 score_format=SCORE_FORMAT,
54 hter_note="Note that FAR = FMR * (1 - FtA), FRR = FtA + FNMR * (1 - FtA) "
55 "and HTER = (FMR + FNMR) / 2",
56 command="bob bio metrics",
57 ),
58 criteria=CRITERIA,
59)
60@common_options.cost_option()
61@click.option(
62 "--score-column",
63 default="score",
64 show_default=True,
65 help=(
66 "Selects the CSV column to consider as scores. This is ignored for "
67 "non-CSV files. The column must contain numerical values."
68 ),
69)
70def metrics(ctx, scores, evaluation, score_column, **kwargs):
71 if "criterion" in ctx.meta and ctx.meta["criterion"] == "rr":
72 process = bio_figure.Metrics(
73 ctx,
74 scores,
75 evaluation,
76 functools.partial(load.cmc, csv_score_column=score_column),
77 )
78 else:
79 process = bio_figure.Metrics(
80 ctx,
81 scores,
82 evaluation,
83 functools.partial(load.split, csv_score_column=score_column),
84 )
85 process.run()
88@common_options.roc_command(
89 common_options.ROC_HELP.format(
90 score_format=SCORE_FORMAT, command="bob bio roc"
91 )
92)
93@click.option(
94 "--score-column",
95 default="score",
96 show_default=True,
97 help=(
98 "Selects the CSV column to consider as scores. This is ignored for "
99 "non-CSV files. The column must contain numerical values."
100 ),
101)
102def roc(ctx, scores, evaluation, score_column, **kwargs):
103 process = bio_figure.Roc(
104 ctx,
105 scores,
106 evaluation,
107 functools.partial(load.split, csv_score_column=score_column),
108 )
109 process.run()
112@common_options.det_command(
113 common_options.DET_HELP.format(
114 score_format=SCORE_FORMAT, command="bob bio det"
115 )
116)
117@click.option(
118 "--score-column",
119 default="score",
120 show_default=True,
121 help=(
122 "Selects the CSV column to consider as scores. This is ignored for "
123 "non-CSV files. The column must contain numerical values."
124 ),
125)
126def det(ctx, scores, evaluation, score_column, **kwargs):
127 process = bio_figure.Det(
128 ctx,
129 scores,
130 evaluation,
131 functools.partial(load.split, csv_score_column=score_column),
132 )
133 process.run()
136@common_options.epc_command(
137 common_options.EPC_HELP.format(
138 score_format=SCORE_FORMAT, command="bob bio epc"
139 )
140)
141@click.option(
142 "--score-column",
143 default="score",
144 show_default=True,
145 help=(
146 "Selects the CSV column to consider as scores. This is ignored for "
147 "non-CSV files. The column must contain numerical values."
148 ),
149)
150def epc(ctx, scores, score_column, **kwargs):
151 process = measure_figure.Epc(
152 ctx,
153 scores,
154 True,
155 functools.partial(load.split, csv_score_column=score_column),
156 )
157 process.run()
160@common_options.hist_command(
161 common_options.HIST_HELP.format(
162 score_format=SCORE_FORMAT, command="bob bio hist"
163 )
164)
165@click.option(
166 "--score-column",
167 default="score",
168 show_default=True,
169 help=(
170 "Selects the CSV column to consider as scores. This is ignored for "
171 "non-CSV files. The column must contain numerical values."
172 ),
173)
174def hist(ctx, scores, evaluation, score_column, **kwargs):
175 process = bio_figure.Hist(
176 ctx,
177 scores,
178 evaluation,
179 functools.partial(load.split, csv_score_column=score_column),
180 )
181 process.run()
184@common_options.evaluate_command(
185 common_options.EVALUATE_HELP.format(
186 score_format=SCORE_FORMAT, command="bob bio evaluate"
187 ),
188 criteria=CRITERIA,
189)
190@click.option(
191 "--score-column",
192 default="score",
193 show_default=True,
194 help=(
195 "NOT YET IMPLEMENTED. "
196 "Selects the CSV column to consider as scores. This is ignored for "
197 "non-CSV files. The column must contain numerical values."
198 ),
199)
200@common_options.cost_option()
201def evaluate(ctx, scores, evaluation, score_column, **kwargs):
202 if score_column != "score":
203 raise NotImplementedError(
204 "'evaluate' does not yet support files with scores in columns "
205 "other than 'score'."
206 )
207 common_options.evaluate_flow(
208 ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs
209 )
212@common_options.multi_metrics_command(
213 common_options.MULTI_METRICS_HELP.format(
214 names="FtA, FAR, FRR, FMR, FNMR, HTER",
215 criteria=CRITERIA,
216 score_format=SCORE_FORMAT,
217 command="bob bio multi-metrics",
218 ),
219 criteria=CRITERIA,
220)
221@click.option(
222 "--score-column",
223 default="score",
224 show_default=True,
225 help=(
226 "Selects the CSV column to consider as scores. This is ignored for "
227 "non-CSV files. The column must contain numerical values."
228 ),
229)
230def multi_metrics(
231 ctx, scores, evaluation, protocols_number, score_column, **kwargs
232):
233 ctx.meta["min_arg"] = protocols_number * (2 if evaluation else 1)
234 process = bio_figure.MultiMetrics(
235 ctx,
236 scores,
237 evaluation,
238 functools.partial(load.split, csv_score_column=score_column),
239 )
240 process.run()
243@click.command()
244@common_options.scores_argument(nargs=-1)
245@common_options.titles_option()
246@common_options.legends_option()
247@common_options.sep_dev_eval_option()
248@common_options.output_plot_file_option(default_out="cmc.pdf")
249@common_options.eval_option()
250@common_options.semilogx_option(True)
251@common_options.axes_val_option(dflt=None)
252@common_options.x_rotation_option()
253@common_options.const_layout_option()
254@common_options.style_option()
255@common_options.linestyles_option()
256@common_options.figsize_option()
257@verbosity_option(logger=logger)
258@click.pass_context
259def cmc(ctx, scores, evaluation, **kwargs):
260 """Plot CMC (cumulative match characteristic curve).
261 graphical presentation of results of an identification task eval, plotting
262 rank values on the x-axis and the probability of correct identification at
263 or below that rank on the y-axis. The values for the axis will be computed
264 using :py:func:`bob.measure.cmc`.
266 You need to provide one or more development score file(s) for each
267 experiment. You can also provide eval files along with dev files. If
268 eval-scores are used, the flag `--eval` must be used. is required
269 in that case. Files must be 4- or 5- columns format, see
270 :py:func:`bob.bio.base.score.load.four_column` and
271 :py:func:`bob.bio.base.score.load.five_column` for details.
274 Examples:
275 $ bob bio cmc -v dev-scores
277 $ bob bio cmc -v dev-scores1 eval-scores1 dev-scores2
278 eval-scores2
280 $ bob bio cmc -v -o my_roc.pdf dev-scores1 eval-scores1
281 """
282 process = bio_figure.Cmc(ctx, scores, evaluation, load.cmc)
283 process.run()
286@click.command()
287@common_options.scores_argument(nargs=-1)
288@common_options.titles_option()
289@common_options.legends_option()
290@common_options.sep_dev_eval_option()
291@common_options.output_plot_file_option(default_out="dir.pdf")
292@common_options.eval_option()
293@common_options.semilogx_option(True)
294@common_options.axes_val_option(dflt=None)
295@common_options.x_rotation_option()
296@rank_option()
297@common_options.const_layout_option()
298@common_options.style_option()
299@common_options.linestyles_option()
300@common_options.figsize_option()
301@common_options.min_far_option()
302@verbosity_option(logger=logger)
303@click.pass_context
304def dir(ctx, scores, evaluation, **kwargs):
305 """Plots the Detection & Identification Rate curve over the FAR.
307 This curve is designed to be used in an open set identification protocol,
308 and defined in Chapter 14.1 of [LiJain2005]_. It requires to have at least
309 one open set probe item, i.e., with no corresponding gallery, such that the
310 positives for that pair are ``None``.
312 The detection and identification curve first computes FAR thresholds based
313 on the out-of-set probe scores (negative scores). For each probe item, the
314 **maximum** negative score is used. Then, it plots the detection and
315 identification rates for those thresholds, which are based on the in-set
316 probe scores only. See [LiJain2005]_ for more details.
318 .. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005
320 You need to provide one or more development score file(s) for each
321 experiment. You can also provide eval files along with dev files. If
322 eval-scores are used, the flag `--eval` must be used. is required
323 in that case. Files must be 4- or 5- columns format, see
324 :py:func:`bob.bio.base.score.load.four_column` and
325 :py:func:`bob.bio.base.score.load.five_column` for details.
327 Examples:
328 $ bob bio dir -e -v dev-scores
330 $ bob bio dir -v dev-scores1 eval-scores1 dev-scores2
331 eval-scores2
333 $ bob bio dir -v -o my_roc.pdf dev-scores1 eval-scores1
334 """
335 process = bio_figure.Dir(ctx, scores, evaluation, load.cmc)
336 process.run()