Coverage for src/bob/measure/script/common_options.py: 92%

543 statements  

« prev     ^ index     » next       coverage.py v7.0.5, created at 2023-06-16 14:10 +0200

1"""Stores click common options for plots""" 

2 

3import functools 

4import logging 

5 

6import click 

7import matplotlib.pyplot as plt 

8import tabulate 

9 

10from clapper.click import verbosity_option 

11from click.types import FLOAT, INT 

12from matplotlib.backends.backend_pdf import PdfPages 

13 

14LOGGER = logging.getLogger(__name__) 

15 

16 

17def bool_option(name, short_name, desc, dflt=False, **kwargs): 

18 """Generic provider for boolean options 

19 

20 Parameters 

21 ---------- 

22 name : str 

23 name of the option 

24 short_name : str 

25 short name for the option 

26 desc : str 

27 short description for the option 

28 dflt : bool or None 

29 Default value 

30 **kwargs 

31 All kwargs are passed to click.option. 

32 

33 Returns 

34 ------- 

35 ``callable`` 

36 A decorator to be used for adding this option. 

37 """ 

38 

39 def custom_bool_option(func): 

40 def callback(ctx, param, value): 

41 ctx.meta[name.replace("-", "_")] = value 

42 return value 

43 

44 return click.option( 

45 "-%s/-n%s" % (short_name, short_name), 

46 "--%s/--no-%s" % (name, name), 

47 default=dflt, 

48 help=desc, 

49 show_default=True, 

50 callback=callback, 

51 is_eager=True, 

52 **kwargs, 

53 )(func) 

54 

55 return custom_bool_option 

56 

57 

58def list_float_option(name, short_name, desc, nitems=None, dflt=None, **kwargs): 

59 """Get option to get a list of float f 

60 

61 Parameters 

62 ---------- 

63 name : str 

64 name of the option 

65 short_name : str 

66 short name for the option 

67 desc : str 

68 short description for the option 

69 nitems : obj:`int`, optional 

70 If given, the parsed list must contains this number of items. 

71 dflt : :any:`list`, optional 

72 List of default values for axes. 

73 **kwargs 

74 All kwargs are passed to click.option. 

75 

76 Returns 

77 ------- 

78 ``callable`` 

79 A decorator to be used for adding this option. 

80 """ 

81 

82 def custom_list_float_option(func): 

83 def callback(ctx, param, value): 

84 if value is None or not value.replace(" ", ""): 

85 value = None 

86 elif value is not None: 

87 tmp = value.split(",") 

88 if nitems is not None and len(tmp) != nitems: 

89 raise click.BadParameter( 

90 "%s Must provide %d axis limits" % (name, nitems) 

91 ) 

92 try: 

93 value = [float(i) for i in tmp] 

94 except Exception: 

95 raise click.BadParameter("Inputs of %s be floats" % name) 

96 ctx.meta[name.replace("-", "_")] = value 

97 return value 

98 

99 return click.option( 

100 "-" + short_name, 

101 "--" + name, 

102 default=dflt, 

103 show_default=True, 

104 help=desc + " Provide just a space (' ') to cancel default values.", 

105 callback=callback, 

106 **kwargs, 

107 )(func) 

108 

109 return custom_list_float_option 

110 

111 

112def open_file_mode_option(**kwargs): 

113 """Get open mode file option 

114 

115 Parameters 

116 ---------- 

117 **kwargs 

118 All kwargs are passed to click.option. 

119 

120 Returns 

121 ------- 

122 ``callable`` 

123 A decorator to be used for adding this option. 

124 """ 

125 

126 def custom_open_file_mode_option(func): 

127 def callback(ctx, param, value): 

128 if value not in ["w", "a", "w+", "a+"]: 

129 raise click.BadParameter("Incorrect open file mode") 

130 ctx.meta["open_mode"] = value 

131 return value 

132 

133 return click.option( 

134 "-om", 

135 "--open-mode", 

136 default="w", 

137 help="File open mode", 

138 callback=callback, 

139 **kwargs, 

140 )(func) 

141 

142 return custom_open_file_mode_option 

143 

144 

145def scores_argument(min_arg=1, force_eval=False, **kwargs): 

146 """Get the argument for scores, and add `dev-scores` and `eval-scores` in 

147 the context when `--eval` flag is on (default) 

148 

149 Parameters 

150 ---------- 

151 min_arg : int 

152 the minimum number of file needed to evaluate a system. For example, 

153 vulnerability analysis needs licit and spoof and therefore min_arg = 2 

154 

155 Returns 

156 ------- 

157 callable 

158 A decorator to be used for adding score arguments for click commands 

159 """ 

160 

161 def custom_scores_argument(func): 

162 def callback(ctx, param, value): 

163 min_a = min_arg or 1 

164 mutli = 1 

165 error = "" 

166 if ( 

167 "evaluation" in ctx.meta and ctx.meta["evaluation"] 

168 ) or force_eval: 

169 mutli += 1 

170 error += "- %d evaluation file(s) \n" % min_a 

171 if "train" in ctx.meta and ctx.meta["train"]: 

172 mutli += 1 

173 error += "- %d training file(s) \n" % min_a 

174 # add more test here if other inputs are needed 

175 

176 min_a *= mutli 

177 ctx.meta["min_arg"] = min_a 

178 if len(value) < 1 or len(value) % ctx.meta["min_arg"] != 0: 

179 raise click.BadParameter( 

180 "The number of provided scores must be > 0 and a multiple of %d " 

181 "because the following files are required:\n" 

182 "- %d development file(s)\n" % (min_a, min_arg or 1) 

183 + error, 

184 ctx=ctx, 

185 ) 

186 ctx.meta["scores"] = value 

187 return value 

188 

189 return click.argument( 

190 "scores", type=click.Path(exists=True), callback=callback, **kwargs 

191 )(func) 

192 

193 return custom_scores_argument 

194 

195 

196def alpha_option(dflt=1, **kwargs): 

197 """An alpha option for plots""" 

198 

199 def custom_eval_option(func): 

200 def callback(ctx, param, value): 

201 ctx.meta["alpha"] = value 

202 return value 

203 

204 return click.option( 

205 "-a", 

206 "--alpha", 

207 default=dflt, 

208 type=click.FLOAT, 

209 show_default=True, 

210 help="Adjusts transparency of plots.", 

211 callback=callback, 

212 **kwargs, 

213 )(func) 

214 

215 return custom_eval_option 

216 

217 

218def no_legend_option(dflt=True, **kwargs): 

219 """Get option flag to say if legend should be displayed or not""" 

220 return bool_option( 

221 "disp-legend", "dl", "If set, no legend will be printed.", dflt=dflt 

222 ) 

223 

224 

225def eval_option(**kwargs): 

226 """Get option flag to say if eval-scores are provided""" 

227 

228 def custom_eval_option(func): 

229 def callback(ctx, param, value): 

230 ctx.meta["evaluation"] = value 

231 return value 

232 

233 return click.option( 

234 "-e", 

235 "--eval", 

236 "evaluation", 

237 is_flag=True, 

238 default=False, 

239 show_default=True, 

240 help="If set, evaluation scores must be provided", 

241 callback=callback, 

242 **kwargs, 

243 )(func) 

244 

245 return custom_eval_option 

246 

247 

248def hide_dev_option(dflt=False, **kwargs): 

249 """Get option flag to say if dev plot should be hidden""" 

250 

251 def custom_hide_dev_option(func): 

252 def callback(ctx, param, value): 

253 ctx.meta["hide_dev"] = value 

254 return value 

255 

256 return click.option( 

257 "--hide-dev", 

258 is_flag=True, 

259 default=dflt, 

260 show_default=True, 

261 help="If set, hide dev related plots", 

262 callback=callback, 

263 **kwargs, 

264 )(func) 

265 

266 return custom_hide_dev_option 

267 

268 

269def sep_dev_eval_option(dflt=True, **kwargs): 

270 """Get option flag to say if dev and eval plots should be in different 

271 plots""" 

272 return bool_option( 

273 "split", 

274 "s", 

275 "If set, evaluation and dev curve in different plots", 

276 dflt, 

277 ) 

278 

279 

280def linestyles_option(dflt=False, **kwargs): 

281 """Get option flag to turn on/off linestyles""" 

282 return bool_option( 

283 "line-styles", 

284 "S", 

285 "If given, applies a different line style to each line.", 

286 dflt, 

287 **kwargs, 

288 ) 

289 

290 

291def cmc_option(**kwargs): 

292 """Get option flag to say if cmc scores""" 

293 return bool_option( 

294 "cmc", "C", "If set, CMC score files are provided", **kwargs 

295 ) 

296 

297 

298def semilogx_option(dflt=False, **kwargs): 

299 """Option to use semilog X-axis""" 

300 return bool_option( 

301 "semilogx", "G", "If set, use semilog on X axis", dflt, **kwargs 

302 ) 

303 

304 

305def tpr_option(dflt=False, **kwargs): 

306 """Option to use TPR (true positive rate) on y-axis""" 

307 return bool_option( 

308 "tpr", 

309 "tpr", 

310 "If set, use TPR (also called 1-FNR, 1-FNMR, or 1-BPCER) on Y axis", 

311 dflt, 

312 **kwargs, 

313 ) 

314 

315 

316def print_filenames_option(dflt=True, **kwargs): 

317 """Option to tell if filenames should be in the title""" 

318 return bool_option( 

319 "show-fn", "P", "If set, show filenames in title", dflt, **kwargs 

320 ) 

321 

322 

323def const_layout_option(dflt=True, **kwargs): 

324 """Option to set matplotlib constrained_layout""" 

325 

326 def custom_layout_option(func): 

327 def callback(ctx, param, value): 

328 ctx.meta["clayout"] = value 

329 plt.rcParams["figure.constrained_layout.use"] = value 

330 return value 

331 

332 return click.option( 

333 "-Y/-nY", 

334 "--clayout/--no-clayout", 

335 default=dflt, 

336 show_default=True, 

337 help="(De)Activate constrained layout", 

338 callback=callback, 

339 **kwargs, 

340 )(func) 

341 

342 return custom_layout_option 

343 

344 

345def axes_val_option(dflt=None, **kwargs): 

346 """Option for setting min/max values on axes""" 

347 return list_float_option( 

348 name="axlim", 

349 short_name="L", 

350 desc="min/max axes values separated by commas (e.g. ``--axlim " 

351 " 0.1,100,0.1,100``)", 

352 nitems=4, 

353 dflt=dflt, 

354 **kwargs, 

355 ) 

356 

357 

358def thresholds_option(**kwargs): 

359 """Option to give a list of thresholds""" 

360 return list_float_option( 

361 name="thres", 

362 short_name="T", 

363 desc="Given threshold for metrics computations, e.g. " 

364 "0.005,0.001,0.056", 

365 nitems=None, 

366 dflt=None, 

367 **kwargs, 

368 ) 

369 

370 

371def lines_at_option(dflt="1e-3", **kwargs): 

372 """Get option to draw const far line""" 

373 return list_float_option( 

374 name="lines-at", 

375 short_name="la", 

376 desc="If given, draw vertical lines at the given axis positions. " 

377 "Your values must be separated with a comma (,) without space. " 

378 "This option works in ROC and DET curves.", 

379 nitems=None, 

380 dflt=dflt, 

381 **kwargs, 

382 ) 

383 

384 

385def x_rotation_option(dflt=0, **kwargs): 

386 """Get option for rotartion of the x axis lables""" 

387 

388 def custom_x_rotation_option(func): 

389 def callback(ctx, param, value): 

390 value = abs(value) 

391 ctx.meta["x_rotation"] = value 

392 return value 

393 

394 return click.option( 

395 "-r", 

396 "--x-rotation", 

397 type=click.INT, 

398 default=dflt, 

399 show_default=True, 

400 help="X axis labels ration", 

401 callback=callback, 

402 **kwargs, 

403 )(func) 

404 

405 return custom_x_rotation_option 

406 

407 

408def legend_ncols_option(dflt=3, **kwargs): 

409 """Get option for number of columns for legends""" 

410 

411 def custom_legend_ncols_option(func): 

412 def callback(ctx, param, value): 

413 value = abs(value) 

414 ctx.meta["legends_ncol"] = value 

415 return value 

416 

417 return click.option( 

418 "-lc", 

419 "--legends-ncol", 

420 type=click.INT, 

421 default=dflt, 

422 show_default=True, 

423 help="The number of columns of the legend layout.", 

424 callback=callback, 

425 **kwargs, 

426 )(func) 

427 

428 return custom_legend_ncols_option 

429 

430 

431def subplot_option(dflt=111, **kwargs): 

432 """Get option to set subplots""" 

433 

434 def custom_subplot_option(func): 

435 def callback(ctx, param, value): 

436 value = abs(value) 

437 nrows = value // 10 

438 nrows, ncols = divmod(nrows, 10) 

439 ctx.meta["n_col"] = ncols 

440 ctx.meta["n_row"] = nrows 

441 return value 

442 

443 return click.option( 

444 "-sp", 

445 "--subplot", 

446 type=click.INT, 

447 default=dflt, 

448 show_default=True, 

449 help="The order of subplots.", 

450 callback=callback, 

451 **kwargs, 

452 )(func) 

453 

454 return custom_subplot_option 

455 

456 

457def cost_option(**kwargs): 

458 """Get option to get cost for FPR""" 

459 

460 def custom_cost_option(func): 

461 def callback(ctx, param, value): 

462 if value < 0 or value > 1: 

463 raise click.BadParameter("Cost for FPR must be betwen 0 and 1") 

464 ctx.meta["cost"] = value 

465 return value 

466 

467 return click.option( 

468 "-C", 

469 "--cost", 

470 type=float, 

471 default=0.99, 

472 show_default=True, 

473 help="Cost for FPR in minDCF", 

474 callback=callback, 

475 **kwargs, 

476 )(func) 

477 

478 return custom_cost_option 

479 

480 

481def points_curve_option(**kwargs): 

482 """Get the number of points use to draw curves""" 

483 

484 def custom_points_curve_option(func): 

485 def callback(ctx, param, value): 

486 if value < 2: 

487 raise click.BadParameter( 

488 "Number of points to draw curves must be greater than 1", 

489 ctx=ctx, 

490 ) 

491 ctx.meta["points"] = value 

492 return value 

493 

494 return click.option( 

495 "-n", 

496 "--points", 

497 type=INT, 

498 default=2000, 

499 show_default=True, 

500 help="The number of points use to draw curves in plots", 

501 callback=callback, 

502 **kwargs, 

503 )(func) 

504 

505 return custom_points_curve_option 

506 

507 

508def n_bins_option(**kwargs): 

509 """Get the number of bins in the histograms""" 

510 possible_strings = [ 

511 "auto", 

512 "fd", 

513 "doane", 

514 "scott", 

515 "rice", 

516 "sturges", 

517 "sqrt", 

518 ] 

519 

520 def custom_n_bins_option(func): 

521 def callback(ctx, param, value): 

522 if value is None: 

523 value = ["doane"] 

524 else: 

525 tmp = value.split(",") 

526 try: 

527 value = [ 

528 int(i) if i not in possible_strings else i for i in tmp 

529 ] 

530 except Exception: 

531 raise click.BadParameter("Incorrect number of bins inputs") 

532 ctx.meta["n_bins"] = value 

533 return value 

534 

535 return click.option( 

536 "-b", 

537 "--nbins", 

538 type=click.STRING, 

539 default="doane", 

540 help="The number of bins for the different quantities to plot, " 

541 "seperated by commas. For example, if you plot histograms " 

542 "of negative and positive scores " 

543 ", input something like `100,doane`. All the " 

544 "possible bin options can be found in https://docs.scipy.org/doc/" 

545 "numpy/reference/generated/numpy.histogram.html. Be aware that " 

546 "for some corner cases, the option `auto` and `fd` can lead to " 

547 "MemoryError.", 

548 callback=callback, 

549 **kwargs, 

550 )(func) 

551 

552 return custom_n_bins_option 

553 

554 

555def table_option(dflt="rst", **kwargs): 

556 """Get table option for tabulate package 

557 More informnations: https://pypi.org/project/tabulate/ 

558 """ 

559 

560 def custom_table_option(func): 

561 def callback(ctx, param, value): 

562 ctx.meta["tablefmt"] = value 

563 return value 

564 

565 return click.option( 

566 "--tablefmt", 

567 type=click.Choice(tabulate.tabulate_formats), 

568 default=dflt, 

569 show_default=True, 

570 help="Format of printed tables.", 

571 callback=callback, 

572 **kwargs, 

573 )(func) 

574 

575 return custom_table_option 

576 

577 

578def output_plot_file_option(default_out="plots.pdf", **kwargs): 

579 """Get options for output file for plots""" 

580 

581 def custom_output_plot_file_option(func): 

582 def callback(ctx, param, value): 

583 """Save ouput file and associated pdf in context list, 

584 print the path of the file in the log""" 

585 ctx.meta["output"] = value 

586 ctx.meta["PdfPages"] = PdfPages(value) 

587 LOGGER.debug("Plots will be output in %s", value) 

588 return value 

589 

590 return click.option( 

591 "-o", 

592 "--output", 

593 default=default_out, 

594 show_default=True, 

595 help="The file to save the plots in.", 

596 callback=callback, 

597 **kwargs, 

598 )(func) 

599 

600 return custom_output_plot_file_option 

601 

602 

603def output_log_metric_option(**kwargs): 

604 """Get options for output file for metrics""" 

605 

606 def custom_output_log_file_option(func): 

607 def callback(ctx, param, value): 

608 if value is not None: 

609 LOGGER.debug("Metrics will be output in %s", value) 

610 ctx.meta["log"] = value 

611 return value 

612 

613 return click.option( 

614 "-l", 

615 "--log", 

616 default=None, 

617 type=click.STRING, 

618 help="If provided, computed numbers are written to " 

619 "this file instead of the standard output.", 

620 callback=callback, 

621 **kwargs, 

622 )(func) 

623 

624 return custom_output_log_file_option 

625 

626 

627def no_line_option(**kwargs): 

628 """Get option flag to say if no line should be displayed""" 

629 

630 def custom_no_line_option(func): 

631 def callback(ctx, param, value): 

632 ctx.meta["no_line"] = value 

633 return value 

634 

635 return click.option( 

636 "--no-line", 

637 is_flag=True, 

638 default=False, 

639 show_default=True, 

640 help="If set does not display vertical lines", 

641 callback=callback, 

642 **kwargs, 

643 )(func) 

644 

645 return custom_no_line_option 

646 

647 

648def criterion_option( 

649 lcriteria=["eer", "min-hter", "far"], check=True, **kwargs 

650): 

651 """Get option flag to tell which criteriom is used (default:eer) 

652 

653 Parameters 

654 ---------- 

655 lcriteria : :any:`list` 

656 List of possible criteria 

657 """ 

658 

659 def custom_criterion_option(func): 

660 list_accepted_crit = ( 

661 lcriteria if lcriteria is not None else ["eer", "min-hter", "far"] 

662 ) 

663 

664 def callback(ctx, param, value): 

665 if value not in list_accepted_crit and check: 

666 raise click.BadParameter( 

667 "Incorrect value for `--criterion`. " 

668 "Must be one of [`%s`]" % "`, `".join(list_accepted_crit) 

669 ) 

670 ctx.meta["criterion"] = value 

671 return value 

672 

673 return click.option( 

674 "-c", 

675 "--criterion", 

676 default="eer", 

677 help="Criterion to compute plots and " 

678 "metrics: %s)" % ", ".join(list_accepted_crit), 

679 callback=callback, 

680 is_eager=True, 

681 show_default=True, 

682 **kwargs, 

683 )(func) 

684 

685 return custom_criterion_option 

686 

687 

688def decimal_option(dflt=1, short="-d", **kwargs): 

689 """Get option to get decimal value""" 

690 

691 def custom_decimal_option(func): 

692 def callback(ctx, param, value): 

693 ctx.meta["decimal"] = value 

694 return value 

695 

696 return click.option( 

697 short, 

698 "--decimal", 

699 type=click.INT, 

700 default=dflt, 

701 help="Number of decimals to be printed.", 

702 callback=callback, 

703 show_default=True, 

704 **kwargs, 

705 )(func) 

706 

707 return custom_decimal_option 

708 

709 

710def far_option(far_name="FAR", **kwargs): 

711 """Get option to get far value""" 

712 

713 def custom_far_option(func): 

714 def callback(ctx, param, value): 

715 if value is not None and (value > 1 or value < 0): 

716 raise click.BadParameter( 

717 "{} value should be between 0 and 1".format(far_name) 

718 ) 

719 ctx.meta["far_value"] = value 

720 return value 

721 

722 return click.option( 

723 "-f", 

724 "--{}-value".format(far_name.lower()), 

725 "far_value", 

726 type=click.FLOAT, 

727 default=None, 

728 help="The {} value for which to compute threshold. This option " 

729 "must be used alongside `--criterion far`.".format(far_name), 

730 callback=callback, 

731 show_default=True, 

732 **kwargs, 

733 )(func) 

734 

735 return custom_far_option 

736 

737 

738def min_far_option(far_name="FAR", dflt=1e-4, **kwargs): 

739 """Get option to get min far value""" 

740 

741 def custom_min_far_option(func): 

742 def callback(ctx, param, value): 

743 if value is not None and (value > 1 or value < 0): 

744 raise click.BadParameter( 

745 "{} value should be between 0 and 1".format(far_name) 

746 ) 

747 ctx.meta["min_far_value"] = value 

748 return value 

749 

750 return click.option( 

751 "-M", 

752 "--min-{}-value".format(far_name.lower()), 

753 "min_far_value", 

754 type=click.FLOAT, 

755 default=dflt, 

756 help="Select the minimum {} value used in ROC and DET plots; " 

757 "should be a power of 10.".format(far_name), 

758 callback=callback, 

759 show_default=True, 

760 **kwargs, 

761 )(func) 

762 

763 return custom_min_far_option 

764 

765 

766def figsize_option(dflt="4,3", **kwargs): 

767 """Get option for matplotlib figsize 

768 

769 Parameters 

770 ---------- 

771 dflt : str 

772 matplotlib default figsize for the command. must be a a list of int 

773 separated by commas. 

774 

775 Returns 

776 ------- 

777 callable 

778 A decorator to be used for adding score arguments for click commands 

779 """ 

780 

781 def custom_figsize_option(func): 

782 def callback(ctx, param, value): 

783 ctx.meta["figsize"] = ( 

784 value if value is None else [float(x) for x in value.split(",")] 

785 ) 

786 if value is not None: 

787 plt.rcParams["figure.figsize"] = ctx.meta["figsize"] 

788 return value 

789 

790 return click.option( 

791 "--figsize", 

792 default=dflt, 

793 show_default=True, 

794 help="If given, will run " 

795 "``plt.rcParams['figure.figsize']=figsize)``. " 

796 "Example: --figsize 4,6", 

797 callback=callback, 

798 **kwargs, 

799 )(func) 

800 

801 return custom_figsize_option 

802 

803 

804def legend_loc_option(dflt="best", **kwargs): 

805 """Get the legend location of the plot""" 

806 

807 def custom_legend_loc_option(func): 

808 def callback(ctx, param, value): 

809 ctx.meta["legend_loc"] = value.replace("-", " ") if value else value 

810 return value 

811 

812 return click.option( 

813 "-ll", 

814 "--legend-loc", 

815 default=dflt, 

816 show_default=True, 

817 type=click.Choice( 

818 [ 

819 "best", 

820 "upper-right", 

821 "upper-left", 

822 "lower-left", 

823 "lower-right", 

824 "right", 

825 "center-left", 

826 "center-right", 

827 "lower-center", 

828 "upper-center", 

829 "center", 

830 ] 

831 ), 

832 help="The legend location code", 

833 callback=callback, 

834 **kwargs, 

835 )(func) 

836 

837 return custom_legend_loc_option 

838 

839 

840def line_width_option(**kwargs): 

841 """Get line width option for the plots""" 

842 

843 def custom_line_width_option(func): 

844 def callback(ctx, param, value): 

845 ctx.meta["line_width"] = value 

846 return value 

847 

848 return click.option( 

849 "--line-width", 

850 type=FLOAT, 

851 help="The line width of plots", 

852 callback=callback, 

853 **kwargs, 

854 )(func) 

855 

856 return custom_line_width_option 

857 

858 

859def marker_style_option(**kwargs): 

860 """Get marker style otpion for the plots""" 

861 

862 def custom_marker_style_option(func): 

863 def callback(ctx, param, value): 

864 ctx.meta["marker_style"] = value 

865 return value 

866 

867 return click.option( 

868 "--marker-style", 

869 type=FLOAT, 

870 help="The marker style of the plots", 

871 callback=callback, 

872 **kwargs, 

873 )(func) 

874 

875 return custom_marker_style_option 

876 

877 

878def legends_option(**kwargs): 

879 """Get the legends option for the different systems""" 

880 

881 def custom_legends_option(func): 

882 def callback(ctx, param, value): 

883 if value is not None: 

884 value = value.split(",") 

885 ctx.meta["legends"] = value 

886 return value 

887 

888 return click.option( 

889 "-lg", 

890 "--legends", 

891 type=click.STRING, 

892 default=None, 

893 help="The legend for each system comma separated. " 

894 "Example: --legends ISV,CNN", 

895 callback=callback, 

896 **kwargs, 

897 )(func) 

898 

899 return custom_legends_option 

900 

901 

902def title_option(**kwargs): 

903 """Get the title option for the different systems""" 

904 

905 def custom_title_option(func): 

906 def callback(ctx, param, value): 

907 ctx.meta["title"] = value 

908 return value 

909 

910 return click.option( 

911 "-t", 

912 "--title", 

913 type=click.STRING, 

914 default=None, 

915 help="The title of the plots. Provide just a space (-t ' ') to " 

916 "remove the titles from figures.", 

917 callback=callback, 

918 **kwargs, 

919 )(func) 

920 

921 return custom_title_option 

922 

923 

924def titles_option(**kwargs): 

925 """Get the titles option for the different plots""" 

926 

927 def custom_title_option(func): 

928 def callback(ctx, param, value): 

929 if value is not None: 

930 value = value.split(",") 

931 ctx.meta["titles"] = value or [] 

932 return value or [] 

933 

934 return click.option( 

935 "-ts", 

936 "--titles", 

937 type=click.STRING, 

938 default=None, 

939 help="The titles of the plots seperated by commas. " 

940 'For example, if the figure has two plots, "MyTitleA,MyTitleB" ' 

941 "is a possible input." 

942 " Provide just a space (-ts ' ') to " 

943 "remove the titles from figures.", 

944 callback=callback, 

945 **kwargs, 

946 )(func) 

947 

948 return custom_title_option 

949 

950 

951def x_label_option(dflt=None, **kwargs): 

952 """Get the label option for X axis""" 

953 

954 def custom_x_label_option(func): 

955 def callback(ctx, param, value): 

956 ctx.meta["x_label"] = value 

957 return value 

958 

959 return click.option( 

960 "-xl", 

961 "--x-label", 

962 type=click.STRING, 

963 default=dflt, 

964 show_default=True, 

965 help="Label for x-axis", 

966 callback=callback, 

967 **kwargs, 

968 )(func) 

969 

970 return custom_x_label_option 

971 

972 

973def y_label_option(dflt=None, **kwargs): 

974 """Get the label option for Y axis""" 

975 

976 def custom_y_label_option(func): 

977 def callback(ctx, param, value): 

978 ctx.meta["y_label"] = value 

979 return value 

980 

981 return click.option( 

982 "-yl", 

983 "--y-label", 

984 type=click.STRING, 

985 default=dflt, 

986 help="Label for y-axis", 

987 callback=callback, 

988 **kwargs, 

989 )(func) 

990 

991 return custom_y_label_option 

992 

993 

994def style_option(**kwargs): 

995 """Get option for matplotlib style""" 

996 

997 def custom_style_option(func): 

998 def callback(ctx, param, value): 

999 ctx.meta["style"] = value 

1000 plt.style.use(value) 

1001 return value 

1002 

1003 return click.option( 

1004 "--style", 

1005 multiple=True, 

1006 type=click.types.Choice(sorted(plt.style.available)), 

1007 help="The matplotlib style to use for plotting. You can provide " 

1008 "multiple styles by repeating this option", 

1009 callback=callback, 

1010 **kwargs, 

1011 )(func) 

1012 

1013 return custom_style_option 

1014 

1015 

1016def metrics_command( 

1017 docstring, 

1018 criteria=("eer", "min-hter", "far"), 

1019 far_name="FAR", 

1020 check_criteria=True, 

1021 **kwarg, 

1022): 

1023 def custom_metrics_command(func): 

1024 func.__doc__ = docstring 

1025 

1026 @click.command(**kwarg) 

1027 @scores_argument(nargs=-1) 

1028 @eval_option() 

1029 @table_option() 

1030 @output_log_metric_option() 

1031 @criterion_option(criteria, check=check_criteria) 

1032 @thresholds_option() 

1033 @far_option(far_name=far_name) 

1034 @legends_option() 

1035 @open_file_mode_option() 

1036 @verbosity_option(LOGGER) 

1037 @click.pass_context 

1038 @decimal_option() 

1039 @functools.wraps(func) 

1040 def wrapper(*args, **kwds): 

1041 return func(*args, **kwds) 

1042 

1043 return wrapper 

1044 

1045 return custom_metrics_command 

1046 

1047 

1048METRICS_HELP = """Prints a table that contains {names} for a given 

1049 threshold criterion ({criteria}). 

1050 {hter_note} 

1051 

1052 You need to provide one or more development score file(s) for each 

1053 experiment. You can also provide evaluation files along with dev files. If 

1054 evaluation scores are provided, you must use flag `--eval`. 

1055 

1056 {score_format} 

1057 

1058 Resulting table format can be changed using the `--tablefmt`. 

1059 

1060 Examples: 

1061 

1062 $ {command} -v scores-dev 

1063 

1064 $ {command} -e -l results.txt sys1/scores-{{dev,eval}} 

1065 

1066 $ {command} -e {{sys1,sys2}}/scores-{{dev,eval}} 

1067 

1068 """ 

1069 

1070 

1071def roc_command(docstring, far_name="FAR"): 

1072 def custom_roc_command(func): 

1073 func.__doc__ = docstring 

1074 

1075 @click.command() 

1076 @scores_argument(nargs=-1) 

1077 @titles_option() 

1078 @legends_option() 

1079 @no_legend_option() 

1080 @legend_loc_option(dflt=None) 

1081 @sep_dev_eval_option() 

1082 @output_plot_file_option(default_out="roc.pdf") 

1083 @eval_option() 

1084 @tpr_option(True) 

1085 @semilogx_option(True) 

1086 @lines_at_option() 

1087 @axes_val_option() 

1088 @min_far_option(far_name=far_name) 

1089 @x_rotation_option() 

1090 @x_label_option() 

1091 @y_label_option() 

1092 @points_curve_option() 

1093 @const_layout_option() 

1094 @figsize_option() 

1095 @style_option() 

1096 @linestyles_option() 

1097 @alpha_option() 

1098 @verbosity_option(LOGGER) 

1099 @click.pass_context 

1100 @functools.wraps(func) 

1101 def wrapper(*args, **kwds): 

1102 return func(*args, **kwds) 

1103 

1104 return wrapper 

1105 

1106 return custom_roc_command 

1107 

1108 

1109ROC_HELP = """Plot ROC (receiver operating characteristic) curve. 

1110 The plot will represent the false match rate on the horizontal axis and the 

1111 false non match rate on the vertical axis. The values for the axis will be 

1112 computed using :py:func:`bob.measure.roc`. 

1113 

1114 You need to provide one or more development score file(s) for each 

1115 experiment. You can also provide evaluation files along with dev files. If 

1116 evaluation scores are provided, you must use flag `--eval`. 

1117 

1118 {score_format} 

1119 

1120 Examples: 

1121 

1122 $ {command} -v scores-dev 

1123 

1124 $ {command} -e -v sys1/scores-{{dev,eval}} 

1125 

1126 $ {command} -e -v -o my_roc.pdf {{sys1,sys2}}/scores-{{dev,eval}} 

1127 """ 

1128 

1129 

1130def det_command(docstring, far_name="FAR"): 

1131 def custom_det_command(func): 

1132 func.__doc__ = docstring 

1133 

1134 @click.command() 

1135 @scores_argument(nargs=-1) 

1136 @output_plot_file_option(default_out="det.pdf") 

1137 @titles_option() 

1138 @legends_option() 

1139 @no_legend_option() 

1140 @legend_loc_option(dflt="upper-right") 

1141 @sep_dev_eval_option() 

1142 @eval_option() 

1143 @axes_val_option(dflt="0.01,95,0.01,95") 

1144 @min_far_option(far_name=far_name) 

1145 @x_rotation_option(dflt=45) 

1146 @x_label_option() 

1147 @y_label_option() 

1148 @points_curve_option() 

1149 @lines_at_option() 

1150 @const_layout_option() 

1151 @figsize_option() 

1152 @style_option() 

1153 @linestyles_option() 

1154 @alpha_option() 

1155 @verbosity_option(LOGGER) 

1156 @click.pass_context 

1157 @functools.wraps(func) 

1158 def wrapper(*args, **kwds): 

1159 return func(*args, **kwds) 

1160 

1161 return wrapper 

1162 

1163 return custom_det_command 

1164 

1165 

1166DET_HELP = """Plot DET (detection error trade-off) curve. 

1167 modified ROC curve which plots error rates on both axes 

1168 (false positives on the x-axis and false negatives on the y-axis). 

1169 

1170 You need to provide one or more development score file(s) for each 

1171 experiment. You can also provide evaluation files along with dev files. If 

1172 evaluation scores are provided, you must use flag `--eval`. 

1173 

1174 {score_format} 

1175 

1176 Examples: 

1177 

1178 $ {command} -v scores-dev 

1179 

1180 $ {command} -e -v sys1/scores-{{dev,eval}} 

1181 

1182 $ {command} -e -v -o my_det.pdf {{sys1,sys2}}/scores-{{dev,eval}} 

1183 """ 

1184 

1185 

1186def epc_command(docstring): 

1187 def custom_epc_command(func): 

1188 func.__doc__ = docstring 

1189 

1190 @click.command() 

1191 @scores_argument(min_arg=1, force_eval=True, nargs=-1) 

1192 @output_plot_file_option(default_out="epc.pdf") 

1193 @titles_option() 

1194 @legends_option() 

1195 @no_legend_option() 

1196 @legend_loc_option(dflt="upper-center") 

1197 @points_curve_option() 

1198 @const_layout_option() 

1199 @x_label_option() 

1200 @y_label_option() 

1201 @figsize_option() 

1202 @style_option() 

1203 @linestyles_option() 

1204 @alpha_option() 

1205 @verbosity_option(LOGGER) 

1206 @click.pass_context 

1207 @functools.wraps(func) 

1208 def wrapper(*args, **kwds): 

1209 return func(*args, **kwds) 

1210 

1211 return wrapper 

1212 

1213 return custom_epc_command 

1214 

1215 

1216EPC_HELP = """Plot EPC (expected performance curve). 

1217 plots the error rate on the eval set depending on a threshold selected 

1218 a-priori on the development set and accounts for varying relative cost 

1219 in [0; 1] of FPR and FNR when calculating the threshold. 

1220 

1221 You need to provide one or more development score and eval file(s) 

1222 for each experiment. 

1223 

1224 {score_format} 

1225 

1226 Examples: 

1227 

1228 $ {command} -v scores-{{dev,eval}} 

1229 

1230 $ {command} -v -o my_epc.pdf {{sys1,sys2}}/scores-{{dev,eval}} 

1231 """ 

1232 

1233 

1234def hist_command(docstring, far_name="FAR"): 

1235 def custom_hist_command(func): 

1236 func.__doc__ = docstring 

1237 

1238 @click.command() 

1239 @scores_argument(nargs=-1) 

1240 @output_plot_file_option(default_out="hist.pdf") 

1241 @eval_option() 

1242 @hide_dev_option() 

1243 @n_bins_option() 

1244 @titles_option() 

1245 @no_legend_option() 

1246 @legend_ncols_option() 

1247 @criterion_option() 

1248 @far_option(far_name=far_name) 

1249 @no_line_option() 

1250 @thresholds_option() 

1251 @subplot_option() 

1252 @const_layout_option() 

1253 @print_filenames_option() 

1254 @figsize_option(dflt=None) 

1255 @style_option() 

1256 @x_label_option() 

1257 @y_label_option() 

1258 @verbosity_option(LOGGER) 

1259 @click.pass_context 

1260 @functools.wraps(func) 

1261 def wrapper(*args, **kwds): 

1262 return func(*args, **kwds) 

1263 

1264 return wrapper 

1265 

1266 return custom_hist_command 

1267 

1268 

1269HIST_HELP = """ Plots histograms of positive and negatives along with threshold 

1270 criterion. 

1271 

1272 You need to provide one or more development score file(s) for each 

1273 experiment. You can also provide evaluation files along with dev files. If 

1274 evaluation scores are provided, you must use the `--eval` flag. The 

1275 threshold is always computed from development score files. 

1276 

1277 By default, when eval-scores are given, only eval-scores histograms are 

1278 displayed with threshold line computed from dev-scores. 

1279 

1280 {score_format} 

1281 

1282 Examples: 

1283 

1284 $ {command} -v scores-dev 

1285 

1286 $ {command} -e -v sys1/scores-{{dev,eval}} 

1287 

1288 $ {command} -e -v --criterion min-hter {{sys1,sys2}}/scores-{{dev,eval}} 

1289 """ 

1290 

1291 

1292def evaluate_command( 

1293 docstring, criteria=("eer", "min-hter", "far"), far_name="FAR" 

1294): 

1295 def custom_evaluate_command(func): 

1296 func.__doc__ = docstring 

1297 

1298 @click.command() 

1299 @scores_argument(nargs=-1) 

1300 @legends_option() 

1301 @sep_dev_eval_option() 

1302 @table_option() 

1303 @eval_option() 

1304 @criterion_option(criteria) 

1305 @far_option(far_name=far_name) 

1306 @output_log_metric_option() 

1307 @output_plot_file_option(default_out="eval_plots.pdf") 

1308 @lines_at_option() 

1309 @points_curve_option() 

1310 @const_layout_option() 

1311 @figsize_option(dflt=None) 

1312 @style_option() 

1313 @linestyles_option() 

1314 @verbosity_option(LOGGER) 

1315 @click.pass_context 

1316 @functools.wraps(func) 

1317 def wrapper(*args, **kwds): 

1318 return func(*args, **kwds) 

1319 

1320 return wrapper 

1321 

1322 return custom_evaluate_command 

1323 

1324 

1325EVALUATE_HELP = """Runs error analysis on score sets. 

1326 

1327 \b 

1328 1. Computes the threshold using a criteria (EER by default) on 

1329 development set scores 

1330 2. Applies the above threshold on evaluation set scores to compute the 

1331 HTER if a eval-score (use --eval) set is provided. 

1332 3. Reports error rates on the console or in a log file. 

1333 4. Plots ROC, DET, and EPC curves and score distributions to a multi-page 

1334 PDF file 

1335 

1336 You need to provide 1 or 2 score files for each biometric system in this 

1337 order: 

1338 

1339 \b 

1340 * development scores 

1341 * evaluation scores 

1342 

1343 {score_format} 

1344 

1345 Examples: 

1346 

1347 $ {command} -v dev-scores 

1348 

1349 $ {command} -v /path/to/sys-{{1,2,3}}/scores-dev 

1350 

1351 $ {command} -e -v /path/to/sys-{{1,2,3}}/scores-{{dev,eval}} 

1352 

1353 $ {command} -v -l metrics.txt -o my_plots.pdf dev-scores 

1354 

1355 This command is a combination of metrics, roc, det, epc, and hist commands. 

1356 If you want more flexibility in your plots, please use the individual 

1357 commands. 

1358 """ 

1359 

1360 

1361def evaluate_flow( 

1362 ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs 

1363): 

1364 # open_mode is always write in this command. 

1365 ctx.meta["open_mode"] = "w" 

1366 criterion = ctx.meta.get("criterion") 

1367 if criterion is not None: 

1368 click.echo("Computing metrics with %s..." % criterion) 

1369 ctx.invoke(metrics, scores=scores, evaluation=evaluation) 

1370 if ctx.meta.get("log") is not None: 

1371 click.echo("[metrics] => %s" % ctx.meta["log"]) 

1372 

1373 # avoid closing pdf file before all figures are plotted 

1374 ctx.meta["closef"] = False 

1375 if evaluation: 

1376 click.echo("Starting evaluate with dev and eval scores...") 

1377 else: 

1378 click.echo("Starting evaluate with dev scores only...") 

1379 click.echo("Computing ROC...") 

1380 # set axes limits for ROC 

1381 ctx.forward(roc) # use class defaults plot settings 

1382 click.echo("Computing DET...") 

1383 ctx.forward(det) # use class defaults plot settings 

1384 if evaluation: 

1385 click.echo("Computing EPC...") 

1386 ctx.forward(epc) # use class defaults plot settings 

1387 # the last one closes the file 

1388 ctx.meta["closef"] = True 

1389 click.echo("Computing score histograms...") 

1390 ctx.meta["criterion"] = "eer" # no criterion passed in evaluate 

1391 ctx.forward(hist) 

1392 click.echo("Evaluate successfully completed!") 

1393 click.echo("[plots] => %s" % (ctx.meta["output"])) 

1394 

1395 

1396def n_protocols_option(required=True, **kwargs): 

1397 """Get option for number of protocols.""" 

1398 

1399 def custom_n_protocols_option(func): 

1400 def callback(ctx, param, value): 

1401 value = abs(value) 

1402 ctx.meta["protocols_number"] = value 

1403 return value 

1404 

1405 return click.option( 

1406 "-pn", 

1407 "--protocols-number", 

1408 type=click.INT, 

1409 show_default=True, 

1410 required=required, 

1411 help="The number of protocols of cross validation.", 

1412 callback=callback, 

1413 **kwargs, 

1414 )(func) 

1415 

1416 return custom_n_protocols_option 

1417 

1418 

1419def multi_metrics_command( 

1420 docstring, criteria=("eer", "min-hter", "far"), far_name="FAR", **kwargs 

1421): 

1422 def custom_metrics_command(func): 

1423 func.__doc__ = docstring 

1424 

1425 @click.command("multi-metrics", **kwargs) 

1426 @scores_argument(nargs=-1) 

1427 @eval_option() 

1428 @n_protocols_option() 

1429 @table_option() 

1430 @output_log_metric_option() 

1431 @criterion_option(criteria) 

1432 @thresholds_option() 

1433 @far_option(far_name=far_name) 

1434 @legends_option() 

1435 @open_file_mode_option() 

1436 @verbosity_option(LOGGER) 

1437 @click.pass_context 

1438 @decimal_option() 

1439 @functools.wraps(func) 

1440 def wrapper(*args, **kwds): 

1441 return func(*args, **kwds) 

1442 

1443 return wrapper 

1444 

1445 return custom_metrics_command 

1446 

1447 

1448MULTI_METRICS_HELP = """Multi protocol (cross-validation) metrics. 

1449 

1450 Prints a table that contains mean and standard deviation of {names} for a given 

1451 threshold criterion ({criteria}). The metrics are averaged over several protocols. 

1452 The idea is that each protocol corresponds to one fold in your cross-validation. 

1453 

1454 You need to provide as many as development score files as the number of 

1455 protocols per system. You can also provide evaluation files along with dev 

1456 files. If evaluation scores are provided, you must use flag `--eval`. The 

1457 number of protocols must be provided using the `--protocols-number` option. 

1458 

1459 {score_format} 

1460 

1461 Resulting table format can be changed using the `--tablefmt`. 

1462 

1463 Examples: 

1464 

1465 $ {command} -vv -pn 3 {{p1,p2,p3}}/scores-dev 

1466 

1467 $ {command} -vv -pn 3 -e {{p1,p2,p3}}/scores-{{dev,eval}} 

1468 

1469 $ {command} -vv -pn 3 -e {{sys1,sys2}}/{{p1,p2,p3}}/scores-{{dev,eval}} 

1470 """