Coverage for /scratch/builds/bob/bob.med.tb/miniconda/conda-bld/bob.med.tb_1637571489937/_test_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placeho/lib/python3.8/site-packages/bob/med/tb/test/test_measures.py: 100%

1#!/usr/bin/env python 

2# -*- coding: utf-8 -*- 

3 

4"""Tests for measure functions""" 

5 

6import random 

7import unittest 

8 

9import numpy 

10 

11from ..utils.measure import ( 

12 base_measures, 

13 bayesian_measures, 

14 beta_credible_region, 

15 get_centered_maxf1 

16) 

17from ..engine.evaluator import sample_measures_for_threshold 

18 

19def test_centered_maxf1(): 

20 

21 # Multiple max F1 

22 f1_scores = numpy.array([0.8, 0.9, 1.0, 1.0, 1.0, 0.3]) 

23 thresholds = numpy.array([0.2, 0.3, 0.4, 0.5, 0.6, 0.7]) 

24 

25 maxf1, threshold = get_centered_maxf1(f1_scores, thresholds) 

26 

27 assert maxf1 == 1.0 

28 assert threshold == 0.5 

29 

30 # Single max F1 

31 f1_scores = numpy.array([0.8, 0.9, 1.0, 0.9, 0.7, 0.3]) 

32 thresholds = numpy.array([0.2, 0.3, 0.4, 0.5, 0.6, 0.7]) 

33 

34 maxf1, threshold = get_centered_maxf1(f1_scores, thresholds) 

35 

36 assert maxf1 == 1.0 

37 assert threshold == 0.4 

38 

39class TestFrequentist(unittest.TestCase): 

40 """ 

41 Unit test for frequentist base measures 

42 """ 

43 

44 def setUp(self): 

45 self.tp = random.randint(1, 100) 

46 self.fp = random.randint(1, 100) 

47 self.tn = random.randint(1, 100) 

48 self.fn = random.randint(1, 100) 

49 

50 def test_precision(self): 

51 precision = base_measures(self.tp, self.fp, self.tn, self.fn)[0] 

52 self.assertEqual((self.tp) / (self.tp + self.fp), precision) 

53 

54 def test_recall(self): 

55 recall = base_measures(self.tp, self.fp, self.tn, self.fn)[1] 

56 self.assertEqual((self.tp) / (self.tp + self.fn), recall) 

57 

58 def test_specificity(self): 

59 specificity = base_measures(self.tp, self.fp, self.tn, self.fn)[2] 

60 self.assertEqual((self.tn) / (self.tn + self.fp), specificity) 

61 

62 def test_accuracy(self): 

63 accuracy = base_measures(self.tp, self.fp, self.tn, self.fn)[3] 

64 self.assertEqual( 

65 (self.tp + self.tn) / (self.tp + self.tn + self.fp + self.fn), 

66 accuracy, 

67 ) 

68 

69 def test_jaccard(self): 

70 jaccard = base_measures(self.tp, self.fp, self.tn, self.fn)[4] 

71 self.assertEqual(self.tp / (self.tp + self.fp + self.fn), jaccard) 

72 

73 def test_f1(self): 

74 p, r, s, a, j, f1 = base_measures(self.tp, self.fp, self.tn, self.fn) 

75 self.assertEqual( 

76 (2.0 * self.tp) / (2.0 * self.tp + self.fp + self.fn), f1 

77 ) 

78 self.assertAlmostEqual((2 * p * r) / (p + r), f1) # base definition 

79 

80 

81class TestBayesian: 

82 """ 

83 Unit test for bayesian base measures 

84 """ 

85 

86 def mean(self, k, l, lambda_): 

87 return (k + lambda_) / (k + l + 2 * lambda_) 

88 

89 def mode1(self, k, l, lambda_): # (k+lambda_), (l+lambda_) > 1 

90 return (k + lambda_ - 1) / (k + l + 2 * lambda_ - 2) 

91 

92 def test_beta_credible_region_base(self): 

93 k = 40 

94 l = 10 

95 lambda_ = 0.5 

96 cover = 0.95 

97 got = beta_credible_region(k, l, lambda_, cover) 

98 # mean, mode, lower, upper 

99 exp = ( 

100 self.mean(k, l, lambda_), 

101 self.mode1(k, l, lambda_), 

102 0.6741731038857685, 

103 0.8922659692341358, 

104 ) 

105 assert numpy.isclose(got, exp).all(), f"{got} <> {exp}" 

106 

107 def test_beta_credible_region_small_k(self): 

108 

109 k = 4 

110 l = 1 

111 lambda_ = 0.5 

112 cover = 0.95 

113 got = beta_credible_region(k, l, lambda_, cover) 

114 # mean, mode, lower, upper 

115 exp = ( 

116 self.mean(k, l, lambda_), 

117 self.mode1(k, l, lambda_), 

118 0.37137359936800574, 

119 0.9774872340008449, 

120 ) 

121 assert numpy.isclose(got, exp).all(), f"{got} <> {exp}" 

122 

123 def test_beta_credible_region_precision_jeffrey(self): 

124 

125 # simulation of situation for precision TP == FP == 0, Jeffrey's prior 

126 k = 0 

127 l = 0 

128 lambda_ = 0.5 

129 cover = 0.95 

130 got = beta_credible_region(k, l, lambda_, cover) 

131 # mean, mode, lower, upper 

132 exp = ( 

133 self.mean(k, l, lambda_), 

134 0.0, 

135 0.0015413331334360135, 

136 0.998458666866564, 

137 ) 

138 assert numpy.isclose(got, exp).all(), f"{got} <> {exp}" 

139 

140 def test_beta_credible_region_precision_flat(self): 

141 

142 # simulation of situation for precision TP == FP == 0, flat prior 

143 k = 0 

144 l = 0 

145 lambda_ = 1.0 

146 cover = 0.95 

147 got = beta_credible_region(k, l, lambda_, cover) 

148 # mean, mode, lower, upper 

149 exp = (self.mean(k, l, lambda_), 0.0, 0.025000000000000022, 0.975) 

150 assert numpy.isclose(got, exp).all(), f"{got} <> {exp}" 

151 

152 def test_bayesian_measures(self): 

153 

154 tp = random.randint(100000, 1000000) 

155 fp = random.randint(100000, 1000000) 

156 tn = random.randint(100000, 1000000) 

157 fn = random.randint(100000, 1000000) 

158 

159 _prec, _rec, _spec, _acc, _jac, _f1 = base_measures(tp, fp, tn, fn) 

160 prec, rec, spec, acc, jac, f1 = bayesian_measures( 

161 tp, fp, tn, fn, 0.5, 0.95 

162 ) 

163 

164 # Notice that for very large k and l, the base frequentist measures 

165 # should be approximately the same as the bayesian mean and mode 

166 # extracted from the beta posterior. We test that here. 

167 assert numpy.isclose( 

168 _prec, prec[0] 

169 ), f"freq: {_prec} <> bays: {prec[0]}" 

170 assert numpy.isclose( 

171 _prec, prec[1] 

172 ), f"freq: {_prec} <> bays: {prec[1]}" 

173 assert numpy.isclose(_rec, rec[0]), f"freq: {_rec} <> bays: {rec[0]}" 

174 assert numpy.isclose(_rec, rec[1]), f"freq: {_rec} <> bays: {rec[1]}" 

175 assert numpy.isclose( 

176 _spec, spec[0] 

177 ), f"freq: {_spec} <> bays: {spec[0]}" 

178 assert numpy.isclose( 

179 _spec, spec[1] 

180 ), f"freq: {_spec} <> bays: {spec[1]}" 

181 assert numpy.isclose(_acc, acc[0]), f"freq: {_acc} <> bays: {acc[0]}" 

182 assert numpy.isclose(_acc, acc[1]), f"freq: {_acc} <> bays: {acc[1]}" 

183 assert numpy.isclose(_jac, jac[0]), f"freq: {_jac} <> bays: {jac[0]}" 

184 assert numpy.isclose(_jac, jac[1]), f"freq: {_jac} <> bays: {jac[1]}" 

185 assert numpy.isclose(_f1, f1[0]), f"freq: {_f1} <> bays: {f1[0]}" 

186 assert numpy.isclose(_f1, f1[1]), f"freq: {_f1} <> bays: {f1[1]}" 

187 

188 # We also test that the interval in question includes the mode and the 

189 # mean in this case. 

190 assert (prec[2] < prec[1]) and ( 

191 prec[1] < prec[3] 

192 ), f"precision is out of bounds {_prec[2]} < {_prec[1]} < {_prec[3]}" 

193 assert (rec[2] < rec[1]) and ( 

194 rec[1] < rec[3] 

195 ), f"recall is out of bounds {_rec[2]} < {_rec[1]} < {_rec[3]}" 

196 assert (spec[2] < spec[1]) and ( 

197 spec[1] < spec[3] 

198 ), f"specif. is out of bounds {_spec[2]} < {_spec[1]} < {_spec[3]}" 

199 assert (acc[2] < acc[1]) and ( 

200 acc[1] < acc[3] 

201 ), f"accuracy is out of bounds {_acc[2]} < {_acc[1]} < {_acc[3]}" 

202 assert (jac[2] < jac[1]) and ( 

203 jac[1] < jac[3] 

204 ), f"jaccard is out of bounds {_jac[2]} < {_jac[1]} < {_jac[3]}" 

205 assert (f1[2] < f1[1]) and ( 

206 f1[1] < f1[3] 

207 ), f"f1-score is out of bounds {_f1[2]} < {_f1[1]} < {_f1[3]}"