Coverage for src/bob/bio/base/script/figure.py: 88%

1"""Plots and measures for bob.bio.base""" 

2 

3import logging 

4import math 

5 

6import click 

7import matplotlib.pyplot as mpl 

8 

9from tabulate import tabulate 

10 

11import bob.measure 

12import bob.measure.script.figure as measure_figure 

13 

14from bob.measure import plot 

15 

16LOGGER = logging.getLogger("bob.bio.base") 

17 

18 

19class Roc(measure_figure.Roc): 

20 def __init__(self, ctx, scores, evaluation, func_load): 

21 super(Roc, self).__init__(ctx, scores, evaluation, func_load) 

22 self._x_label = ctx.meta.get("x_label") or "FMR" 

23 default_y_label = "1 - FNMR" if self._tpr else "FNMR" 

24 self._y_label = ctx.meta.get("y_label") or default_y_label 

25 

26 

27class Det(measure_figure.Det): 

28 def __init__(self, ctx, scores, evaluation, func_load): 

29 super(Det, self).__init__(ctx, scores, evaluation, func_load) 

30 self._x_label = ctx.meta.get("x_label") or "FMR (%)" 

31 self._y_label = ctx.meta.get("y_label") or "FNMR (%)" 

32 

33 

34class Cmc(measure_figure.PlotBase): 

35 """Handles the plotting of Cmc""" 

36 

37 def __init__(self, ctx, scores, evaluation, func_load): 

38 super(Cmc, self).__init__(ctx, scores, evaluation, func_load) 

39 self._semilogx = ctx.meta.get("semilogx", True) 

40 self._titles = self._titles or ["CMC dev.", "CMC eval."] 

41 self._x_label = self._x_label or "Rank" 

42 self._y_label = self._y_label or "Identification rate" 

43 self._max_R = 0 

44 

45 def compute(self, idx, input_scores, input_names): 

46 """Plot CMC for dev and eval data using 

47 :py:func:`bob.measure.plot.cmc`""" 

48 mpl.figure(1) 

49 if self._eval: 

50 linestyle = "-" if not self._split else self._linestyles[idx] 

51 LOGGER.info("CMC dev. curve using %s", input_names[0]) 

52 rank = plot.cmc( 

53 input_scores[0], 

54 logx=self._semilogx, 

55 color=self._colors[idx], 

56 linestyle=linestyle, 

57 label=self._label("dev.", idx), 

58 ) 

59 self._max_R = max(rank, self._max_R) 

60 linestyle = "--" 

61 if self._split: 

62 mpl.figure(2) 

63 linestyle = self._linestyles[idx] 

64 

65 LOGGER.info("CMC eval. curve using %s", input_names[1]) 

66 rank = plot.cmc( 

67 input_scores[1], 

68 logx=self._semilogx, 

69 color=self._colors[idx], 

70 linestyle=linestyle, 

71 label=self._label("eval.", idx), 

72 ) 

73 self._max_R = max(rank, self._max_R) 

74 else: 

75 LOGGER.info("CMC dev. curve using %s", input_names[0]) 

76 rank = plot.cmc( 

77 input_scores[0], 

78 logx=self._semilogx, 

79 color=self._colors[idx], 

80 linestyle=self._linestyles[idx], 

81 label=self._label("dev.", idx), 

82 ) 

83 self._max_R = max(rank, self._max_R) 

84 

85 

86class Dir(measure_figure.PlotBase): 

87 """Handles the plotting of DIR curve""" 

88 

89 def __init__(self, ctx, scores, evaluation, func_load): 

90 super(Dir, self).__init__(ctx, scores, evaluation, func_load) 

91 self._semilogx = ctx.meta.get("semilogx", True) 

92 self._rank = ctx.meta.get("rank", 1) 

93 self._titles = self._titles or ["DIR curve"] * 2 

94 self._x_label = self._x_label or "False Positive Identification Rate" 

95 self._y_label = self._y_label or "True Positive Identification Rate" 

96 

97 def compute(self, idx, input_scores, input_names): 

98 """Plot DIR for dev and eval data using 

99 :py:func:`bob.measure.plot.detection_identification_curve`""" 

100 mpl.figure(1) 

101 if self._eval: 

102 linestyle = "-" if not self._split else self._linestyles[idx] 

103 LOGGER.info("DIR dev. curve using %s", input_names[0]) 

104 plot.detection_identification_curve( 

105 input_scores[0], 

106 rank=self._rank, 

107 logx=self._semilogx, 

108 color=self._colors[idx], 

109 linestyle=linestyle, 

110 label=self._label("dev", idx), 

111 ) 

112 linestyle = "--" 

113 if self._split: 

114 mpl.figure(2) 

115 linestyle = self._linestyles[idx] 

116 

117 LOGGER.info("DIR eval. curve using %s", input_names[1]) 

118 plot.detection_identification_curve( 

119 input_scores[1], 

120 rank=self._rank, 

121 logx=self._semilogx, 

122 color=self._colors[idx], 

123 linestyle=linestyle, 

124 label=self._label("eval", idx), 

125 ) 

126 else: 

127 LOGGER.info("DIR dev. curve using %s", input_names[0]) 

128 plot.detection_identification_curve( 

129 input_scores[0], 

130 rank=self._rank, 

131 logx=self._semilogx, 

132 color=self._colors[idx], 

133 linestyle=self._linestyles[idx], 

134 label=self._label("dev", idx), 

135 ) 

136 

137 if self._min_dig is not None: 

138 mpl.xlim(xmin=math.pow(10, self._min_dig)) 

139 

140 

141class Metrics(measure_figure.Metrics): 

142 """Compute metrics from score files""" 

143 

144 def __init__( 

145 self, 

146 ctx, 

147 scores, 

148 evaluation, 

149 func_load, 

150 names=( 

151 "Failure to Acquire", 

152 "False Match Rate", 

153 "False Non Match Rate", 

154 "False Accept Rate", 

155 "False Reject Rate", 

156 "Half Total Error Rate", 

157 ), 

158 ): 

159 super(Metrics, self).__init__(ctx, scores, evaluation, func_load, names) 

160 

161 def init_process(self): 

162 if self._criterion == "rr": 

163 self._thres = ( 

164 [None] * self.n_systems if self._thres is None else self._thres 

165 ) 

166 

167 def compute(self, idx, input_scores, input_names): 

168 """Compute metrics for the given criteria""" 

169 title = self._legends[idx] if self._legends is not None else None 

170 headers = ["" or title, "Dev. %s" % input_names[0]] 

171 if self._eval and input_scores[1] is not None: 

172 headers.append("eval % s" % input_names[1]) 

173 if self._criterion == "rr": 

174 rr = bob.measure.recognition_rate(input_scores[0], self._thres[idx]) 

175 dev_rr = "%.1f%%" % (100 * rr) 

176 raws = [["RR", dev_rr]] 

177 if self._eval and input_scores[1] is not None: 

178 rr = bob.measure.recognition_rate( 

179 input_scores[1], self._thres[idx] 

180 ) 

181 eval_rr = "%.1f%%" % (100 * rr) 

182 raws[0].append(eval_rr) 

183 click.echo( 

184 tabulate(raws, headers, self._tablefmt), file=self.log_file 

185 ) 

186 elif self._criterion == "mindcf": 

187 if "cost" in self._ctx.meta: 

188 cost = self._ctx.meta.get("cost", 0.99) 

189 threshold = ( 

190 bob.measure.min_weighted_error_rate_threshold( 

191 input_scores[0][0], input_scores[0][1], cost 

192 ) 

193 if self._thres is None 

194 else self._thres[idx] 

195 ) 

196 if self._thres is None: 

197 click.echo( 

198 "[minDCF - Cost:%f] Threshold on Development set `%s`: %e" 

199 % (cost, input_names[0], threshold), 

200 file=self.log_file, 

201 ) 

202 else: 

203 click.echo( 

204 "[minDCF] User defined Threshold: %e" % threshold, 

205 file=self.log_file, 

206 ) 

207 # apply threshold to development set 

208 far, frr = bob.measure.farfrr( 

209 input_scores[0][0], input_scores[0][1], threshold 

210 ) 

211 dev_far_str = "%.1f%%" % (100 * far) 

212 dev_frr_str = "%.1f%%" % (100 * frr) 

213 dev_mindcf_str = "%.1f%%" % ( 

214 (cost * far + (1 - cost) * frr) * 100.0 

215 ) 

216 raws = [ 

217 ["FAR", dev_far_str], 

218 ["FRR", dev_frr_str], 

219 ["minDCF", dev_mindcf_str], 

220 ] 

221 if self._eval and input_scores[1] is not None: 

222 # apply threshold to development set 

223 far, frr = bob.measure.farfrr( 

224 input_scores[1][0], input_scores[1][1], threshold 

225 ) 

226 eval_far_str = "%.1f%%" % (100 * far) 

227 eval_frr_str = "%.1f%%" % (100 * frr) 

228 eval_mindcf_str = "%.1f%%" % ( 

229 (cost * far + (1 - cost) * frr) * 100.0 

230 ) 

231 raws[0].append(eval_far_str) 

232 raws[1].append(eval_frr_str) 

233 raws[2].append(eval_mindcf_str) 

234 click.echo( 

235 tabulate(raws, headers, self._tablefmt), file=self.log_file 

236 ) 

237 elif self._criterion == "cllr": 

238 cllr = bob.measure.calibration.cllr( 

239 input_scores[0][0], input_scores[0][1] 

240 ) 

241 min_cllr = bob.measure.calibration.min_cllr( 

242 input_scores[0][0], input_scores[0][1] 

243 ) 

244 dev_cllr_str = "%.1f%%" % cllr 

245 dev_min_cllr_str = "%.1f%%" % min_cllr 

246 raws = [["Cllr", dev_cllr_str], ["minCllr", dev_min_cllr_str]] 

247 if self._eval and input_scores[1] is not None: 

248 cllr = bob.measure.calibration.cllr( 

249 input_scores[1][0], input_scores[1][1] 

250 ) 

251 min_cllr = bob.measure.calibration.min_cllr( 

252 input_scores[1][0], input_scores[1][1] 

253 ) 

254 eval_cllr_str = "%.1f%%" % cllr 

255 eval_min_cllr_str = "%.1f%%" % min_cllr 

256 raws[0].append(eval_cllr_str) 

257 raws[1].append(eval_min_cllr_str) 

258 click.echo( 

259 tabulate(raws, headers, self._tablefmt), file=self.log_file 

260 ) 

261 else: 

262 title = self._legends[idx] if self._legends is not None else None 

263 all_metrics = self._get_all_metrics(idx, input_scores, input_names) 

264 headers = [" " or title, "Development"] 

265 rows = [ 

266 [self.names[0], all_metrics[0][0]], 

267 [self.names[1], all_metrics[0][1]], 

268 [self.names[2], all_metrics[0][2]], 

269 [self.names[3], all_metrics[0][3]], 

270 [self.names[4], all_metrics[0][4]], 

271 [self.names[5], all_metrics[0][5]], 

272 ] 

273 

274 if self._eval: 

275 # computes statistics for the eval set based on the threshold a 

276 # priori 

277 headers.append("Evaluation") 

278 rows[0].append(all_metrics[1][0]) 

279 rows[1].append(all_metrics[1][1]) 

280 rows[2].append(all_metrics[1][2]) 

281 rows[3].append(all_metrics[1][3]) 

282 rows[4].append(all_metrics[1][4]) 

283 rows[5].append(all_metrics[1][5]) 

284 

285 click.echo( 

286 tabulate(rows, headers, self._tablefmt), file=self.log_file 

287 ) 

288 

289 

290class MultiMetrics(measure_figure.MultiMetrics): 

291 """Compute metrics from score files""" 

292 

293 def __init__(self, ctx, scores, evaluation, func_load): 

294 super(MultiMetrics, self).__init__( 

295 ctx, 

296 scores, 

297 evaluation, 

298 func_load, 

299 names=( 

300 "Failure to Acquire", 

301 "False Match Rate", 

302 "False Non Match Rate", 

303 "False Accept Rate", 

304 "False Reject Rate", 

305 "Half Total Error Rate", 

306 ), 

307 ) 

308 

309 

310class Hist(measure_figure.Hist): 

311 """Histograms for biometric scores""" 

312 

313 def _setup_hist(self, neg, pos): 

314 self._title_base = "Biometric scores" 

315 self._density_hist(pos[0], n=0, label="Genuines", alpha=0.9, color="C2") 

316 self._density_hist( 

317 neg[0], n=1, label="Zero-effort impostors", alpha=0.8, color="C0" 

318 )