Coverage for /scratch/builds/bob/bob.ip.binseg/miniconda/conda-bld/bob.ip.binseg_1673966692152/_test_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_p/lib/python3.10/site-packages/bob/ip/common/script/evaluate.py: 88%

1#!/usr/bin/env python 

2# coding=utf-8 

3 

4import logging 

5 

6logger = logging.getLogger(__name__) 

7 

8 

9def base_evaluate( 

10 output_folder, 

11 predictions_folder, 

12 dataset, 

13 second_annotator, 

14 overlayed, 

15 threshold, 

16 steps, 

17 parallel, 

18 detection, 

19 **kwargs, 

20): 

21 """Create base evaluate function for segmentation / detection tasks.""" 

22 if detection: 

23 from ...detect.engine.evaluator import compare_annotators, run 

24 else: 

25 from ...binseg.engine.evaluator import compare_annotators, run 

26 

27 def _validate_threshold(t, dataset): 

28 """Validate the user threshold selection. Returns parsed threshold.""" 

29 if t is None: 

30 return 0.5 

31 

32 try: 

33 # we try to convert it to float first 

34 t = float(t) 

35 if t < 0.0 or t > 1.0: 

36 raise ValueError( 

37 "Float thresholds must be within range [0.0, 1.0]" 

38 ) 

39 except ValueError: 

40 # it is a bit of text - assert dataset with name is available 

41 if not isinstance(dataset, dict): 

42 raise ValueError( 

43 "Threshold should be a floating-point number " 

44 "if your provide only a single dataset for evaluation" 

45 ) 

46 if t not in dataset: 

47 raise ValueError( 

48 f"Text thresholds should match dataset names, " 

49 f"but {t} is not available among the datasets provided (" 

50 f"({', '.join(dataset.keys())})" 

51 ) 

52 

53 return t 

54 

55 threshold = _validate_threshold(threshold, dataset) 

56 

57 if not isinstance(dataset, dict): 

58 dataset = {"test": dataset} 

59 

60 if second_annotator is None: 

61 second_annotator = {} 

62 elif not isinstance(second_annotator, dict): 

63 second_annotator = {"test": second_annotator} 

64 # else, second_annotator must be a dict 

65 

66 if isinstance(threshold, str): 

67 # first run evaluation for reference dataset, do not save overlays 

68 logger.info(f"Evaluating threshold on '{threshold}' set") 

69 threshold = run( 

70 dataset[threshold], threshold, predictions_folder, steps=steps 

71 ) 

72 logger.info(f"Set --threshold={threshold:.5f}") 

73 

74 # clean-up the overlayed path 

75 if overlayed is not None: 

76 overlayed = overlayed.strip() 

77 

78 # now run with the 

79 for k, v in dataset.items(): 

80 if k.startswith("_"): 

81 logger.info(f"Skipping dataset '{k}' (not to be evaluated)") 

82 continue 

83 logger.info(f"Analyzing '{k}' set...") 

84 run( 

85 v, 

86 k, 

87 predictions_folder, 

88 output_folder, 

89 overlayed, 

90 threshold, 

91 steps=steps, 

92 parallel=parallel, 

93 ) 

94 second = second_annotator.get(k) 

95 if second is not None: 

96 if not second.all_keys_match(v): 

97 logger.warning( 

98 f"Key mismatch between `dataset[{k}]` and " 

99 f"`second_annotator[{k}]` - skipping " 

100 f"second-annotator comparisons for {k} subset" 

101 ) 

102 else: 

103 compare_annotators( 

104 v, second, k, output_folder, overlayed, parallel=parallel 

105 )