Coverage for /scratch/builds/bob/bob.ip.binseg/miniconda/conda-bld/bob.ip.binseg_1673966692152/_test_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_p/lib/python3.10/site-packages/bob/ip/common/test/test

1#!/usr/bin/env python

2# coding=utf-8

4"""Tests for our CLI applications"""

6import fnmatch

7import logging

8import os

9import re

10import tempfile

12from click.testing import CliRunner

14from bob.extension.scripts.click_helper import assert_click_runner_result

16from . import mock_dataset

18stare_datadir, stare_dataset = mock_dataset()

21def _assert_exit_0(result):

23 assert_click_runner_result(result)

26def _check_help(entry_point):

28 runner = CliRunner()

29 result = runner.invoke(entry_point, ["--help"])

30 _assert_exit_0(result)

31 assert result.output.startswith("Usage:")

34def test_main_help_binseg():

35 from ..script.common import binseg

37 _check_help(binseg)

40def test_main_help_detect():

41 from ..script.common import detect

43 _check_help(detect)

46def test_binseg_experiment_help():

47 from ...binseg.script.experiment import experiment

49 _check_help(experiment)

52def test_detect_experiment_help():

53 from ...detect.script.experiment import experiment

55 _check_help(experiment)

58def _str_counter(substr, s):

59 return sum(1 for _ in re.finditer(substr, s, re.MULTILINE))

62def _check_experiment_stare(caplog, overlay, multiprocess=False, extra_valid=0):

64 from ...binseg.script.experiment import experiment

66 # ensures we capture only ERROR messages and above by default

67 caplog.set_level(logging.ERROR)

69 runner = CliRunner()

70 with runner.isolated_filesystem(), caplog.at_level(

71 logging.INFO, logger="bob.ip.binseg"

72 ), tempfile.NamedTemporaryFile(mode="wt") as config:

74 # re-write STARE dataset configuration for test

75 config.write("from bob.ip.binseg.data.stare import _make_dataset\n")

76 config.write(f"_raw = _make_dataset('{stare_datadir}')\n")

77 config.write(

78 "from bob.ip.binseg.configs.datasets.stare import _maker\n"

79 )

80 config.write("dataset = _maker('ah', _raw)\n")

81 if extra_valid > 0:

82 # simulates the existence of a single extra validation dataset

83 # which is simply a copy of the __valid__ dataset for this test...

84 config.write(

85 f"dataset['__extra_valid__'] = "

86 f"{extra_valid}*[dataset['__valid__']]\n"

87 )

88 config.write("second_annotator = _maker('vk', _raw)\n")

89 config.flush()

91 output_folder = "results"

92 options = [

93 "lwnet",

94 config.name,

95 "-vv",

96 "--epochs=1",

97 "--batch-size=1",

98 "--steps=10",

99 f"--output-folder={output_folder}",

100 "--monitoring-interval=2",

101 "--plot-limits=0.1",

102 "1.0",

103 "0.1",

104 "1.0",

105 ]

106 if overlay:

107 options += ["--overlayed"]

108 if multiprocess:

109 options += ["--parallel=1"]

110

111 result = runner.invoke(experiment, options)

112 _assert_exit_0(result)

113

114 # check command-line

115 assert os.path.exists(os.path.join(output_folder, "command.sh"))

116

117 # check model was saved

118 train_folder = os.path.join(output_folder, "model")

119 assert os.path.exists(

120 os.path.join(train_folder, "model_final_epoch.pth")

121 )

122 assert os.path.exists(

123 os.path.join(train_folder, "model_lowest_valid_loss.pth")

124 )

125 assert os.path.exists(os.path.join(train_folder, "last_checkpoint"))

126 assert os.path.exists(os.path.join(train_folder, "constants.csv"))

127 assert os.path.exists(os.path.join(train_folder, "trainlog.csv"))

128 assert os.path.exists(os.path.join(train_folder, "model_summary.txt"))

129

130 # check predictions are there

131 predict_folder = os.path.join(output_folder, "predictions")

132 traindir = os.path.join(predict_folder, "train", "stare-images")

133 assert os.path.exists(traindir)

134 assert len(fnmatch.filter(os.listdir(traindir), "*.hdf5")) == 10

135 testdir = os.path.join(predict_folder, "test", "stare-images")

136 assert os.path.exists(testdir)

137 assert len(fnmatch.filter(os.listdir(testdir), "*.hdf5")) == 10

138

139 overlay_folder = os.path.join(output_folder, "overlayed", "predictions")

140 traindir = os.path.join(overlay_folder, "train", "stare-images")

141 testdir = os.path.join(overlay_folder, "test", "stare-images")

142 if overlay:

143 # check overlayed images are there (since we requested them)

144 assert os.path.exists(traindir)

145 assert len(fnmatch.filter(os.listdir(traindir), "*.png")) == 10

146 # check overlayed images are there (since we requested them)

147 assert os.path.exists(testdir)

148 assert len(fnmatch.filter(os.listdir(testdir), "*.png")) == 10

149 else:

150 assert not os.path.exists(traindir)

151 assert not os.path.exists(testdir)

152

153 # check evaluation outputs

154 eval_folder = os.path.join(output_folder, "analysis")

155 assert os.path.exists(os.path.join(eval_folder, "train.csv"))

156 # checks individual performance figures are there

157 traindir = os.path.join(eval_folder, "train", "stare-images")

158 assert os.path.exists(traindir)

159 assert len(fnmatch.filter(os.listdir(traindir), "*.csv")) == 10

160

161 assert os.path.exists(os.path.join(eval_folder, "test.csv"))

162 # checks individual performance figures are there

163 testdir = os.path.join(eval_folder, "test", "stare-images")

164 assert os.path.exists(testdir)

165 assert len(fnmatch.filter(os.listdir(testdir), "*.csv")) == 10

166

167 assert os.path.exists(

168 os.path.join(eval_folder, "second-annotator", "train.csv")

169 )

170 # checks individual performance figures are there

171 traindir_sa = os.path.join(

172 eval_folder, "second-annotator", "train", "stare-images"

173 )

174 assert os.path.exists(traindir_sa)

175 assert len(fnmatch.filter(os.listdir(traindir_sa), "*.csv")) == 10

176

177 assert os.path.exists(

178 os.path.join(eval_folder, "second-annotator", "test.csv")

179 )

180 testdir_sa = os.path.join(

181 eval_folder, "second-annotator", "test", "stare-images"

182 )

183 assert os.path.exists(testdir_sa)

184 assert len(fnmatch.filter(os.listdir(testdir_sa), "*.csv")) == 10

185

186 overlay_folder = os.path.join(output_folder, "overlayed", "analysis")

187 traindir = os.path.join(overlay_folder, "train", "stare-images")

188 testdir = os.path.join(overlay_folder, "test", "stare-images")

189 if overlay:

190 # check overlayed images are there (since we requested them)

191 assert os.path.exists(traindir)

192 assert len(fnmatch.filter(os.listdir(traindir), "*.png")) == 10

193 assert os.path.exists(testdir)

194 assert len(fnmatch.filter(os.listdir(testdir), "*.png")) == 10

195 else:

196 assert not os.path.exists(traindir)

197 assert not os.path.exists(testdir)

198

199 # check overlayed images from first-to-second annotator comparisons

200 # are there (since we requested them)

201 overlay_folder = os.path.join(

202 output_folder, "overlayed", "analysis", "second-annotator"

203 )

204 traindir = os.path.join(overlay_folder, "train", "stare-images")

205 testdir = os.path.join(overlay_folder, "test", "stare-images")

206 if overlay:

207 assert os.path.exists(traindir)

208 assert len(fnmatch.filter(os.listdir(traindir), "*.png")) == 10

209 assert os.path.exists(testdir)

210 assert len(fnmatch.filter(os.listdir(testdir), "*.png")) == 10

211 else:

212 assert not os.path.exists(traindir)

213 assert not os.path.exists(testdir)

214

215 # check outcomes of the comparison phase

216 assert os.path.exists(os.path.join(output_folder, "comparison.pdf"))

217 assert os.path.exists(os.path.join(output_folder, "comparison.rst"))

218

219 keywords = {

220 r"^Started training$": 1,

221 r"^Found $dedicated$ '__train__' set for training$": 1,

222 r"^Found $dedicated$ '__valid__' set for validation$": 1,

223 r"^Will checkpoint lowest loss model on validation set$": 1,

224 f"^Found {extra_valid} extra validation": 1 if extra_valid else 0,

225 r"^Extra validation sets are NOT used for model checkpointing": 1

226 if extra_valid

227 else 0,

228 r"^Continuing from epoch 0$": 1,

229 r"^Saving model summary at.*$": 1,

230 r"^Model has.*$": 1,

231 r"^Found new low on validation set.*$": 1,

232 r"^Saving checkpoint": 2,

233 r"^Ended training$": 1,

234 r"^Started prediction$": 1,

235 r"^Loading checkpoint from": 1,

236 r"^Ended prediction$": 1,

237 r"^Started evaluation$": 1,

238 r"^Maximum F1-score of.*$chosen \*a posteriori\*$$": 3,

239 r"^F1-score of.*$chosen \*a priori\*$$": 2,

240 r"^F1-score of.*$second annotator; threshold=0.5$$": 2,

241 r"^Ended evaluation$": 1,

242 r"^Started comparison$": 1,

243 r"^Loading measures from": 4,

244 r"^Creating and saving plot at": 1,

245 r"^Tabulating performance summary...": 1,

246 r"^Saving table at": 1,

247 r"^Ended comparison.*$": 1,

248 }

249 messages = "\n".join([k.getMessage() for k in caplog.records])

250 for k, v in keywords.items():

251 total = _str_counter(k, messages)

252 assert total == v, (

253 f"message '{k}' appears {total} times, but I expected "

254 f"it to appear {v} times"

255 )

256

257

258def test_experiment_stare_with_overlay(caplog):

259 _check_experiment_stare(caplog, overlay=True)

260

261

262def test_experiment_stare_without_overlay(caplog):

263 _check_experiment_stare(caplog, overlay=False)

264

265

266def test_experiment_stare_with_multiprocessing(caplog):

267 _check_experiment_stare(caplog, overlay=False, multiprocess=True)

268

269

270def test_experiment_stare_with_extra_validation(caplog):

271 _check_experiment_stare(caplog, overlay=False, extra_valid=1)

272

273

274def test_experiment_stare_with_multiple_extra_validation(caplog):

275 _check_experiment_stare(caplog, overlay=False, extra_valid=3)

276

277

278def _check_experiment_stare_detection(

279 caplog, overlay, multiprocess=False, extra_valid=0

280):

281

282 from ...detect.script.experiment import experiment

283

284 # ensures we capture only ERROR messages and above by default

285 caplog.set_level(logging.ERROR)

286

287 runner = CliRunner()

288 with runner.isolated_filesystem(), caplog.at_level(

289 logging.INFO, logger="bob.ip.detect"

290 ), tempfile.NamedTemporaryFile(mode="wt") as config:

291

292 # re-write STARE dataset configuration for test

293 config.write("from bob.ip.binseg.data.stare import _make_dataset\n")

294 config.write(f"_raw = _make_dataset('{stare_datadir}')\n")

295 config.write(

296 "from bob.ip.detect.configs.datasets.stare import _maker\n"

297 )

298 config.write("dataset = _maker('ah', _raw)\n")

299 if extra_valid > 0:

300 # simulates the existence of a single extra validation dataset

301 # which is simply a copy of the __valid__ dataset for this test...

302 config.write(

303 f"dataset['__extra_valid__'] = "

304 f"{extra_valid}*[dataset['__valid__']]\n"

305 )

306 config.flush()

307

308 output_folder = "results"

309 options = [

310 "faster-rcnn",

311 config.name,

312 "-vv",

313 "--epochs=1",

314 "--batch-size=1",

315 "--steps=10",

316 f"--output-folder={output_folder}",

317 "--monitoring-interval=2",

318 ]

319 if overlay:

320 options += ["--overlayed"]

321 if multiprocess:

322 options += ["--parallel=1"]

323

324 result = runner.invoke(experiment, options)

325 _assert_exit_0(result)

326

327 # check command-line

328 assert os.path.exists(os.path.join(output_folder, "command.sh"))

329

330 # check model was saved

331 train_folder = os.path.join(output_folder, "model")

332 assert os.path.exists(

333 os.path.join(train_folder, "model_final_epoch.pth")

334 )

335 assert os.path.exists(

336 os.path.join(train_folder, "model_lowest_valid_loss.pth")

337 )

338 assert os.path.exists(os.path.join(train_folder, "last_checkpoint"))

339 assert os.path.exists(os.path.join(train_folder, "constants.csv"))

340 assert os.path.exists(os.path.join(train_folder, "trainlog.csv"))

341 assert os.path.exists(os.path.join(train_folder, "model_summary.txt"))

342

343 # check predictions are there

344 predict_folder = os.path.join(output_folder, "predictions")

345 traindir = os.path.join(predict_folder, "train", "stare-images")

346 assert os.path.exists(traindir)

347 assert len(fnmatch.filter(os.listdir(traindir), "*.hdf5")) == 10

348 testdir = os.path.join(predict_folder, "test", "stare-images")

349 assert os.path.exists(testdir)

350 assert len(fnmatch.filter(os.listdir(testdir), "*.hdf5")) == 10

351

352 overlay_folder = os.path.join(output_folder, "overlayed", "predictions")

353 traindir = os.path.join(overlay_folder, "train", "stare-images")

354 testdir = os.path.join(overlay_folder, "test", "stare-images")

355 if overlay:

356 # check overlayed images are there (since we requested them)

357 assert os.path.exists(traindir)

358 assert len(fnmatch.filter(os.listdir(traindir), "*.png")) == 10

359 # check overlayed images are there (since we requested them)

360 assert os.path.exists(testdir)

361 assert len(fnmatch.filter(os.listdir(testdir), "*.png")) == 10

362 else:

363 assert not os.path.exists(traindir)

364 assert not os.path.exists(testdir)

365

366 # check evaluation outputs

367 eval_folder = os.path.join(output_folder, "analysis")

368 assert os.path.exists(os.path.join(eval_folder, "train.csv"))

369 # checks individual performance figures are there

370 traindir = os.path.join(eval_folder, "train", "stare-images")

371 assert os.path.exists(traindir)

372 assert len(fnmatch.filter(os.listdir(traindir), "*.csv")) == 10

373

374 assert os.path.exists(os.path.join(eval_folder, "test.csv"))

375 # checks individual performance figures are there

376 testdir = os.path.join(eval_folder, "test", "stare-images")

377 assert os.path.exists(testdir)

378 assert len(fnmatch.filter(os.listdir(testdir), "*.csv")) == 10

379

380 overlay_folder = os.path.join(output_folder, "overlayed", "analysis")

381 traindir = os.path.join(overlay_folder, "train", "stare-images")

382 testdir = os.path.join(overlay_folder, "test", "stare-images")

383 if overlay:

384 # check overlayed images are there (since we requested them)

385 assert os.path.exists(traindir)

386 assert len(fnmatch.filter(os.listdir(traindir), "*.png")) == 10

387 assert os.path.exists(testdir)

388 assert len(fnmatch.filter(os.listdir(testdir), "*.png")) == 10

389 else:

390 assert not os.path.exists(traindir)

391 assert not os.path.exists(testdir)

392

393 # check overlayed images from first-to-second annotator comparisons

394 # are there (since we requested them)

395 traindir = os.path.join(overlay_folder, "train", "stare-images")

396 testdir = os.path.join(overlay_folder, "test", "stare-images")

397 if overlay:

398 assert os.path.exists(traindir)

399 assert len(fnmatch.filter(os.listdir(traindir), "*.png")) == 10

400 assert os.path.exists(testdir)

401 assert len(fnmatch.filter(os.listdir(testdir), "*.png")) == 10

402 else:

403 assert not os.path.exists(traindir)

404 assert not os.path.exists(testdir)

405

406 keywords = {

407 r"^Started training$": 1,

408 r"^Found $dedicated$ '__train__' set for training$": 1,

409 r"^Found $dedicated$ '__valid__' set for validation$": 1,

410 r"^Will checkpoint lowest loss model on validation set$": 1,

411 f"^Found {extra_valid} extra validation": 1 if extra_valid else 0,

412 r"^Extra validation sets are NOT used for model checkpointing": 1

413 if extra_valid

414 else 0,

415 r"^Continuing from epoch 0$": 1,

416 r"^Saving model summary at.*$": 1,

417 r"^Model has.*$": 1,

418 r"^Found new low on validation set.*$": 1,

419 r"^Saving checkpoint": 2,

420 r"^Ended training$": 1,

421 r"^Started prediction$": 1,

422 r"^Loading checkpoint from": 1,

423 r"^Ended prediction$": 1,

424 r"^Started evaluation$": 1,

425 r"^Maximum IoU of.*$chosen \*a posteriori\*$$": 3,

426 r"^IoU of.*$chosen \*a priori\*$$": 2,

427 r"^Ended evaluation$": 1,

428 r"^Started comparison$": 1,

429 r"^Loading measures from": 2,

430 r"^Tabulating performance summary...": 1,

431 r"^Saving table at": 1,

432 r"^Ended comparison.*$": 1,

433 }

434 messages = "\n".join([k.getMessage() for k in caplog.records])

435 for k, v in keywords.items():

436 total = _str_counter(k, messages)

437 assert total == v, (

438 f"message '{k}' appears {total} times, but I expected "

439 f"it to appear {v} times"

440 )

441

442

443def test_experiment_stare_with_overlay_detection(caplog):

444 _check_experiment_stare_detection(caplog, overlay=True)

445

446

447def test_experiment_stare_without_overlay_detection(caplog):

448 _check_experiment_stare_detection(caplog, overlay=False)

449

450

451def test_experiment_stare_with_multiprocessing_detection(caplog):

452 _check_experiment_stare_detection(caplog, overlay=False, multiprocess=True)

453

454

455def test_experiment_stare_with_extra_validation_detection(caplog):

456 _check_experiment_stare_detection(caplog, overlay=False, extra_valid=1)

457

458

459def _check_train(caplog, runner):

460

461 from ...binseg.script.train import train

462

463 with tempfile.NamedTemporaryFile(mode="wt") as config, caplog.at_level(

464 logging.INFO, logger="bob.ip.binseg"

465 ):

466

467 # single training set configuration

468 config.write("from bob.ip.binseg.data.stare import _make_dataset\n")

469 config.write(f"_raw = _make_dataset('{stare_datadir}')\n")

470 config.write(

471 "from bob.ip.binseg.configs.datasets.stare import _maker\n"

472 )

473 config.write("dataset = _maker('ah', _raw)\n")

474 config.flush()

475

476 output_folder = "results"

477 result = runner.invoke(

478 train,

479 [

480 "lwnet",

481 config.name,

482 "-vv",

483 "--epochs=1",

484 "--batch-size=1",

485 f"--output-folder={output_folder}",

486 ],

487 )

488 _assert_exit_0(result)

489

490 assert os.path.exists(

491 os.path.join(output_folder, "model_final_epoch.pth")

492 )

493 assert os.path.exists(

494 os.path.join(output_folder, "model_lowest_valid_loss.pth")

495 )

496 assert os.path.exists(os.path.join(output_folder, "last_checkpoint"))

497 assert os.path.exists(os.path.join(output_folder, "constants.csv"))

498 assert os.path.exists(os.path.join(output_folder, "trainlog.csv"))

499 assert os.path.exists(os.path.join(output_folder, "model_summary.txt"))

500

501 keywords = {

502 r"^Found $dedicated$ '__train__' set for training$": 1,

503 r"^Found $dedicated$ '__valid__' set for validation$": 1,

504 r"^Continuing from epoch 0$": 1,

505 r"^Saving model summary at.*$": 1,

506 r"^Model has.*$": 1,

507 r"^Saving checkpoint to .*/model_lowest_valid_loss.pth$": 1,

508 r"^Saving checkpoint to .*/model_final_epoch.pth$": 1,

509 r"^Total training time:": 1,

510 }

511

512 messages = "\n".join([k.getMessage() for k in caplog.records])

513 for k, v in keywords.items():

514 total = _str_counter(k, messages)

515 assert total == v, (

516 f"message '{k}' appears {total} times, but I expected "

517 f"it to appear {v} times"

518 )

519

520

521def _check_predict(caplog, runner):

522

523 from ...binseg.script.predict import predict

524

525 with tempfile.NamedTemporaryFile(mode="wt") as config, caplog.at_level(

526 logging.INFO, logger="bob.ip.binseg"

527 ):

528

529 # single training set configuration

530 config.write("from bob.ip.binseg.data.stare import _make_dataset\n")

531 config.write(f"_raw = _make_dataset('{stare_datadir}')\n")

532 config.write(

533 "from bob.ip.binseg.configs.datasets.stare import _maker\n"

534 )

535 config.write("dataset = _maker('ah', _raw)['test']\n")

536 config.flush()

537

538 output_folder = "predictions"

539 overlay_folder = os.path.join("overlayed", "predictions")

540 result = runner.invoke(

541 predict,

542 [

543 "lwnet",

544 config.name,

545 "-vv",

546 "--batch-size=1",

547 "--weight=results/model_final_epoch.pth",

548 f"--output-folder={output_folder}",

549 f"--overlayed={overlay_folder}",

550 ],

551 )

552 _assert_exit_0(result)

553

554 # check predictions are there

555 basedir = os.path.join(output_folder, "test", "stare-images")

556 assert os.path.exists(basedir)

557 assert len(fnmatch.filter(os.listdir(basedir), "*.hdf5")) == 10

558

559 # check overlayed images are there (since we requested them)

560 basedir = os.path.join(overlay_folder, "test", "stare-images")

561 assert os.path.exists(basedir)

562 assert len(fnmatch.filter(os.listdir(basedir), "*.png")) == 10

563

564 keywords = {

565 r"^Loading checkpoint from.*$": 1,

566 r"^Total time:.*$": 1,

567 }

568

569 messages = "\n".join([k.getMessage() for k in caplog.records])

570 for k, v in keywords.items():

571 total = _str_counter(k, messages)

572 assert total == v, (

573 f"message '{k}' appears {total} times, but I expected "

574 f"it to appear {v} times"

575 )

576

577

578def _check_evaluate(caplog, runner):

579

580 from ...binseg.script.evaluate import evaluate

581

582 with tempfile.NamedTemporaryFile(mode="wt") as config, caplog.at_level(

583 logging.INFO, logger="bob.ip.binseg"

584 ):

585

586 # single training set configuration

587 config.write("from bob.ip.binseg.data.stare import _make_dataset\n")

588 config.write(f"_raw = _make_dataset('{stare_datadir}')\n")

589 config.write(

590 "from bob.ip.binseg.configs.datasets.stare import _maker\n"

591 )

592 config.write("dataset = _maker('ah', _raw)['test']\n")

593 config.write("second_annotator = _maker('vk', _raw)['test']\n")

594 config.flush()

595

596 output_folder = "evaluations"

597 overlay_folder = os.path.join("overlayed", "analysis")

598 result = runner.invoke(

599 evaluate,

600 [

601 config.name,

602 "-vv",

603 "--steps=10",

604 f"--output-folder={output_folder}",

605 "--predictions-folder=predictions",

606 f"--overlayed={overlay_folder}",

607 ],

608 )

609 _assert_exit_0(result)

610

611 assert os.path.exists(os.path.join(output_folder, "test.csv"))

612 # checks individual performance figures are there

613 testdir = os.path.join(output_folder, "test", "stare-images")

614 assert os.path.exists(testdir)

615 assert len(fnmatch.filter(os.listdir(testdir), "*.csv")) == 10

616

617 assert os.path.exists(

618 os.path.join(output_folder, "second-annotator", "test.csv")

619 )

620 # checks individual performance figures are there

621 testdir_sa = os.path.join(

622 output_folder, "second-annotator", "test", "stare-images"

623 )

624 assert os.path.exists(testdir_sa)

625 assert len(fnmatch.filter(os.listdir(testdir_sa), "*.csv")) == 10

626

627 # check overlayed images are there (since we requested them)

628 basedir = os.path.join(overlay_folder, "test", "stare-images")

629 assert os.path.exists(basedir)

630 assert len(fnmatch.filter(os.listdir(basedir), "*.png")) == 10

631

632 keywords = {

633 r"^Maximum F1-score of.*$chosen \*a posteriori\*$$": 1,

634 r"^F1-score of.*$chosen \*a priori\*$$": 1,

635 r"^F1-score of.*$second annotator; threshold=0.5$$": 1,

636 }

637

638 messages = "\n".join([k.getMessage() for k in caplog.records])

639 for k, v in keywords.items():

640 total = _str_counter(k, messages)

641 assert total == v, (

642 f"message '{k}' appears {total} times, but I expected "

643 f"it to appear {v} times"

644 )

645

646

647def _check_compare(caplog, runner):

648

649 from ...binseg.script.compare import compare

650

651 with caplog.at_level(logging.INFO, logger="bob.ip.binseg"):

652

653 output_folder = "evaluations"

654 result = runner.invoke(

655 compare,

656 [

657 "-vv",

658 # label - path to measures

659 "test",

660 os.path.join(output_folder, "test.csv"),

661 "test (2nd. human)",

662 os.path.join(output_folder, "second-annotator", "test.csv"),

663 "--output-figure=comparison.pdf",

664 "--output-table=comparison.rst",

665 ],

666 )

667 _assert_exit_0(result)

668

669 assert os.path.exists("comparison.pdf")

670 assert os.path.exists("comparison.rst")

671

672 keywords = {

673 r"^Loading measures from": 2,

674 r"^Creating and saving plot at": 1,

675 r"^Tabulating performance summary...": 1,

676 r"^Saving table at": 1,

677 }

678 messages = "\n".join([k.getMessage() for k in caplog.records])

679 for k, v in keywords.items():

680 total = _str_counter(k, messages)

681 assert total == v, (

682 f"message '{k}' appears {total} times, but I expected "

683 f"it to appear {v} times"

684 )

685

686

687def _check_significance(caplog, runner):

688

689 from ...binseg.script.significance import significance

690

691 with tempfile.NamedTemporaryFile(mode="wt") as config, caplog.at_level(

692 logging.INFO, logger="bob.ip.binseg"

693 ):

694

695 config.write("from bob.ip.binseg.data.stare import _make_dataset\n")

696 config.write(f"_raw = _make_dataset('{stare_datadir}')\n")

697 config.write(

698 "from bob.ip.binseg.configs.datasets.stare import _maker\n"

699 )

700 config.write("dataset = _maker('ah', _raw)\n")

701 config.flush()

702

703 ofolder = "significance"

704 cfolder = os.path.join(ofolder, "caches")

705

706 result = runner.invoke(

707 significance,

708 [

709 "-vv",

710 config.name,

711 "--names=v1",

712 "v2",

713 "--predictions=predictions",

714 "predictions",

715 "--threshold=0.5",

716 "--size=64",

717 "64",

718 "--stride=32",

719 "32",

720 "--figure=accuracy",

721 f"--output-folder={ofolder}",

722 f"--checkpoint-folder={cfolder}",

723 ],

724 )

725 _assert_exit_0(result)

726

727 assert os.path.exists(ofolder)

728 assert os.path.exists(cfolder)

729 assert os.path.exists(os.path.join(ofolder, "analysis.pdf"))

730 assert os.path.exists(os.path.join(ofolder, "analysis.txt"))

731

732 keywords = {

733 r"^Evaluating sliding window 'accuracy' on": 2,

734 r"^Evaluating sliding window 'accuracy' differences on": 1,

735 # r"^Basic statistics from distributions:$": 1,

736 r"^Writing analysis figures": 1,

737 r"^Writing analysis summary": 1,

738 r"^Differences are exactly zero": 2,

739 }

740 messages = "\n".join([k.getMessage() for k in caplog.records])

741 for k, v in keywords.items():

742 total = _str_counter(k, messages)

743 assert total == v, (

744 f"message '{k}' appears {total} times, but I expected "

745 f"it to appear {v} times"

746 )

747

748

749def test_discrete_experiment_stare(caplog):

750

751 # ensures we capture only ERROR messages and above by default

752 caplog.set_level(logging.ERROR)

753

754 runner = CliRunner()

755 with runner.isolated_filesystem():

756 _check_train(caplog, runner)

757 _check_predict(caplog, runner)

758 _check_evaluate(caplog, runner)

759 _check_compare(caplog, runner)

760 # _check_significance(caplog, runner)

761

762

763def _check_train_detection(caplog, runner):

764

765 from ...detect.script.train import train

766

767 with tempfile.NamedTemporaryFile(mode="wt") as config, caplog.at_level(

768 logging.INFO, logger="bob.ip.detect"

769 ):

770

771 # single training set configuration

772 config.write("from bob.ip.binseg.data.stare import _make_dataset\n")

773 config.write(f"_raw = _make_dataset('{stare_datadir}')\n")

774 config.write(

775 "from bob.ip.detect.configs.datasets.stare import _maker\n"

776 )

777 config.write("dataset = _maker('ah', _raw)\n")

778 config.flush()

779

780 output_folder = "results"

781 result = runner.invoke(

782 train,

783 [

784 "faster-rcnn",

785 config.name,

786 "-vv",

787 "--epochs=1",

788 "--batch-size=1",

789 f"--output-folder={output_folder}",

790 ],

791 )

792 _assert_exit_0(result)

793

794 assert os.path.exists(

795 os.path.join(output_folder, "model_final_epoch.pth")

796 )

797 assert os.path.exists(

798 os.path.join(output_folder, "model_lowest_valid_loss.pth")

799 )

800 assert os.path.exists(os.path.join(output_folder, "last_checkpoint"))

801 assert os.path.exists(os.path.join(output_folder, "constants.csv"))

802 assert os.path.exists(os.path.join(output_folder, "trainlog.csv"))

803 assert os.path.exists(os.path.join(output_folder, "model_summary.txt"))

804

805 keywords = {

806 r"^Found $dedicated$ '__train__' set for training$": 1,

807 r"^Found $dedicated$ '__valid__' set for validation$": 1,

808 r"^Continuing from epoch 0$": 1,

809 r"^Saving model summary at.*$": 1,

810 r"^Model has.*$": 1,

811 r"^Saving checkpoint to .*/model_lowest_valid_loss.pth$": 1,

812 r"^Saving checkpoint to .*/model_final_epoch.pth$": 1,

813 r"^Total training time:": 1,

814 }

815

816 messages = "\n".join([k.getMessage() for k in caplog.records])

817 for k, v in keywords.items():

818 total = _str_counter(k, messages)

819 assert total == v, (

820 f"message '{k}' appears {total} times, but I expected "

821 f"it to appear {v} times"

822 )

823

824

825def _check_predict_detection(caplog, runner):

826

827 from ...detect.script.predict import predict

828

829 with tempfile.NamedTemporaryFile(mode="wt") as config, caplog.at_level(

830 logging.INFO, logger="bob.ip.detect"

831 ):

832

833 # single training set configuration

834 config.write("from bob.ip.binseg.data.stare import _make_dataset\n")

835 config.write(f"_raw = _make_dataset('{stare_datadir}')\n")

836 config.write(

837 "from bob.ip.detect.configs.datasets.stare import _maker\n"

838 )

839 config.write("dataset = _maker('ah', _raw)['test']\n")

840 config.flush()

841

842 output_folder = "predictions"

843 overlay_folder = os.path.join("overlayed", "predictions")

844 result = runner.invoke(

845 predict,

846 [

847 "faster-rcnn",

848 config.name,

849 "-vv",

850 "--batch-size=1",

851 "--weight=results/model_final_epoch.pth",

852 f"--output-folder={output_folder}",

853 f"--overlayed={overlay_folder}",

854 ],

855 )

856 _assert_exit_0(result)

857

858 # check predictions are there

859 basedir = os.path.join(output_folder, "test", "stare-images")

860 assert os.path.exists(basedir)

861 assert len(fnmatch.filter(os.listdir(basedir), "*.hdf5")) == 10

862

863 # check overlayed images are there (since we requested them)

864 basedir = os.path.join(overlay_folder, "test", "stare-images")

865 assert os.path.exists(basedir)

866 assert len(fnmatch.filter(os.listdir(basedir), "*.png")) == 10

867

868 keywords = {

869 r"^Loading checkpoint from.*$": 1,

870 r"^Total time:.*$": 1,

871 }

872

873 messages = "\n".join([k.getMessage() for k in caplog.records])

874 for k, v in keywords.items():

875 total = _str_counter(k, messages)

876 assert total == v, (

877 f"message '{k}' appears {total} times, but I expected "

878 f"it to appear {v} times"

879 )

880

881

882def _check_evaluate_detection(caplog, runner):

883

884 from ...detect.script.evaluate import evaluate

885

886 with tempfile.NamedTemporaryFile(mode="wt") as config, caplog.at_level(

887 logging.INFO, logger="bob.ip.detect"

888 ):

889

890 # single training set configuration

891 config.write("from bob.ip.binseg.data.stare import _make_dataset\n")

892 config.write(f"_raw = _make_dataset('{stare_datadir}')\n")

893 config.write(

894 "from bob.ip.detect.configs.datasets.stare import _maker\n"

895 )

896 config.write("dataset = _maker('ah', _raw)['test']\n")

897 config.flush()

898

899 output_folder = "evaluations"

900 overlay_folder = os.path.join("overlayed", "analysis")

901 result = runner.invoke(

902 evaluate,

903 [

904 config.name,

905 "-vv",

906 "--steps=10",

907 f"--output-folder={output_folder}",

908 "--predictions-folder=predictions",

909 f"--overlayed={overlay_folder}",

910 ],

911 )

912 _assert_exit_0(result)

913

914 assert os.path.exists(os.path.join(output_folder, "test.csv"))

915 # checks individual performance figures are there

916 testdir = os.path.join(output_folder, "test", "stare-images")

917 assert os.path.exists(testdir)

918 assert len(fnmatch.filter(os.listdir(testdir), "*.csv")) == 10

919

920 # check overlayed images are there (since we requested them)

921 basedir = os.path.join(overlay_folder, "test", "stare-images")

922 assert os.path.exists(basedir)

923 assert len(fnmatch.filter(os.listdir(basedir), "*.png")) == 10

924

925 keywords = {

926 r"^Maximum IoU of.*$chosen \*a posteriori\*$$": 1,

927 r"^IoU of.*$chosen \*a priori\*$$": 1,

928 }

929

930 messages = "\n".join([k.getMessage() for k in caplog.records])

931 for k, v in keywords.items():

932 total = _str_counter(k, messages)

933 assert total == v, (

934 f"message '{k}' appears {total} times, but I expected "

935 f"it to appear {v} times"

936 )

937

938

939def test_discrete_experiment_stare_detection(caplog):

940

941 # ensures we capture only ERROR messages and above by default

942 caplog.set_level(logging.ERROR)

943

944 runner = CliRunner()

945 with runner.isolated_filesystem():

946 _check_train_detection(caplog, runner)

947 _check_predict_detection(caplog, runner)

948 _check_evaluate_detection(caplog, runner)

949

950

951def test_train_help():

952 from ...binseg.script.train import train

953

954 _check_help(train)

955

956

957def test_detect_train_help():

958 from ...detect.script.train import train

959

960 _check_help(train)

961

962

963def test_predict_help():

964 from ...binseg.script.predict import predict

965

966 _check_help(predict)

967

968

969def test_detect_predict_help():

970 from ...detect.script.predict import predict

971

972 _check_help(predict)

973

974

975def test_evaluate_help():

976 from ...binseg.script.evaluate import evaluate

977

978 _check_help(evaluate)

979

980

981def test_detect_evaluate_help():

982 from ...detect.script.evaluate import evaluate

983

984 _check_help(evaluate)

985

986

987def test_compare_help():

988 from ...binseg.script.compare import compare

989

990 _check_help(compare)

991

992

993def test_detect_compare_help():

994 from ...detect.script.compare import compare

995

996 _check_help(compare)

997

998

999def test_mkmask_help():

1000 from ...binseg.script.mkmask import mkmask

1001

1002 _check_help(mkmask)

1003

1004

1005def test_significance_help():

1006 from ...binseg.script.significance import significance

1007

1008 _check_help(significance)

1009

1010

1011def test_config_help():

1012 from ...binseg.script.config import config

1013

1014 _check_help(config)

1015

1016

1017def test_detect_config_help():

1018 from ...detect.script.config import config

1019

1020 _check_help(config)

1021

1022

1023def test_config_list_help():

1024 from ...binseg.script.config import list

1025

1026 _check_help(list)

1027

1028

1029def test_detect_config_list_help():

1030 from ...detect.script.config import list

1031

1032 _check_help(list)

1033

1034

1035def test_config_list():

1036 from ...binseg.script.config import list

1037

1038 runner = CliRunner()

1039 result = runner.invoke(list)

1040 _assert_exit_0(result)

1041 assert "module: bob.ip.binseg.configs.datasets" in result.output

1042 assert "module: bob.ip.binseg.configs.models" in result.output

1043

1044

1045def test_detect_config_list():

1046 from ...detect.script.config import list

1047

1048 runner = CliRunner()

1049 result = runner.invoke(list)

1050 _assert_exit_0(result)

1051 assert "module: bob.ip.detect.configs.datasets" in result.output

1052 assert "module: bob.ip.detect.configs.models" in result.output

1053

1054

1055def test_config_list_v():

1056 from ...binseg.script.config import list

1057

1058 runner = CliRunner()

1059 result = runner.invoke(list, ["--verbose"])

1060 _assert_exit_0(result)

1061 assert "module: bob.ip.binseg.configs.datasets" in result.output

1062 assert "module: bob.ip.binseg.configs.models" in result.output

1063

1064

1065def test_config_describe_help():

1066 from ...binseg.script.config import describe

1067

1068 _check_help(describe)

1069

1070

1071def test_detect_config_describe_help():

1072 from ...detect.script.config import describe

1073

1074 _check_help(describe)

1075

1076

1077def test_config_describe_drive():

1078 from ...binseg.script.config import describe

1079

1080 runner = CliRunner()

1081 result = runner.invoke(describe, ["drive"])

1082 _assert_exit_0(result)

1083 assert "[DRIVE-2004]" in result.output

1084

1085

1086def test_config_copy_help():

1087 from ...binseg.script.config import copy

1088

1089 _check_help(copy)

1090

1091

1092def test_config_copy():

1093 from ...binseg.script.config import copy

1094

1095 runner = CliRunner()

1096 with runner.isolated_filesystem():

1097 result = runner.invoke(copy, ["drive", "test.py"])

1098 _assert_exit_0(result)

1099 with open("test.py") as f:

1100 data = f.read()

1101 assert "[DRIVE-2004]" in data

1102

1103

1104def test_detect_config_copy_help():

1105 from ...detect.script.config import copy

1106

1107 _check_help(copy)

1108

1109

1110def test_dataset_help():

1111 from ...binseg.script.dataset import dataset

1112

1113 _check_help(dataset)

1114

1115

1116def test_dataset_list_help():

1117 from ...binseg.script.dataset import list

1118

1119 _check_help(list)

1120

1121

1122def test_dataset_list():

1123 from ...binseg.script.dataset import list

1124

1125 runner = CliRunner()

1126 result = runner.invoke(list)

1127 _assert_exit_0(result)

1128 assert result.output.startswith("Supported datasets:")

1129

1130

1131def test_dataset_check_help():

1132 from ...binseg.script.dataset import check

1133

1134 _check_help(check)

1135

1136

1137def test_dataset_check():

1138 from ...binseg.script.dataset import check

1139

1140 runner = CliRunner()

1141 result = runner.invoke(check, ["--verbose", "--verbose", "--limit=2"])

1142 _assert_exit_0(result)

1143

1144

1145def test_detect_dataset_help():

1146 from ...detect.script.dataset import dataset

1147

1148 _check_help(dataset)

1149

1150

1151def test_detect_dataset_list_help():

1152 from ...detect.script.dataset import list

1153

1154 _check_help(list)

1155

1156

1157def test_detect_dataset_list():

1158 from ...detect.script.dataset import list

1159

1160 runner = CliRunner()

1161 result = runner.invoke(list)

1162 _assert_exit_0(result)

1163 assert result.output.startswith("Supported datasets:")

1164

1165

1166def test_detect_dataset_check_help():

1167 from ...detect.script.dataset import check

1168

1169 _check_help(check)

1170

1171

1172def test_detect_dataset_check():

1173 from ...detect.script.dataset import check

1174

1175 runner = CliRunner()

1176 result = runner.invoke(check, ["--verbose", "--verbose", "--limit=2"])

1177 _assert_exit_0(result)