Coverage for CIResults/tests/test_run_import.py: 100%

1from unittest.mock import patch, Mock, call, mock_open, MagicMock 

2from django.core.management import call_command 

3from django.test import TestCase 

4from django.utils import timezone 

5from io import StringIO 

6 

7from CIResults.models import Component, Build, Test, Machine, RunConfig, TestSuite, TestsuiteRun 

8from CIResults.models import TextStatus, TestResult, KnownFailure, RunFilterStatistic, UnknownFailure 

9from CIResults.models import IssueFilter, Issue, IssueFilterAssociated 

10from CIResults.run_import import TestsuiteRunResults, TestsuiteTestResult, TestsuiteResults 

11from CIResults.run_import import RunConfigResults, PiglitResult, TestSuiteRunDef, issue_simple_stats_recomputing 

12from add_build import BuildResult 

13 

14import configparser 

15import datetime 

16 

17 

18class BuildResultTests(TestCase): 

19 

20 def __mock_configparser__(self): 

21 # Replace the configparser instanciated by RunConfigResult with our own 

22 self.config = configparser.ConfigParser() 

23 patcher_api_call = patch('configparser.ConfigParser') 

24 mock_matches = patcher_api_call.start() 

25 mock_matches.return_value = self.config 

26 self.addCleanup(patcher_api_call.stop) 

27 

28 # Make the read() function a noop 

29 self.config.read = lambda x: 0 

30 

31 def test_parse_run_info__empty(self): 

32 self.__mock_configparser__() 

33 

34 # Check empty 

35 self.config.read_string("") 

36 self.assertRaisesMessage(ValueError, 

37 "The build info file build.ini is invalid: missing the section CIRESULTS_BUILD", 

38 BuildResult.from_build_ini, "") 

39 

40 def test_parse_run_info__more_sections(self): 

41 self.__mock_configparser__() 

42 

43 cfg = """[CIRESULTS_BUILD] 

44 field = value 

45 

46 [SECTION] 

47 field = value""" 

48 self.config.read_string(cfg) 

49 msg = "The build info file build.ini is invalid: only the section CIRESULTS_BUILD is allowed" 

50 self.assertRaisesMessage(ValueError, msg, BuildResult.from_build_ini, "") 

51 

52 def test_parse_run_info__minimal(self): 

53 self.__mock_configparser__() 

54 

55 component = Component.objects.create(name="COMPONENT1", description="desc", 

56 url="url", public=True) 

57 

58 cfg = """[CIRESULTS_BUILD] 

59 name: BUILD2 

60 component: COMPONENT1 

61 version: VERSION1""" 

62 self.config.read_string(cfg) 

63 build = BuildResult.from_build_ini("") 

64 build.commit_to_db() 

65 

66 obj = Build.objects.get(name="BUILD2") 

67 self.assertEqual(obj.name, "BUILD2") 

68 self.assertEqual(obj.component, component) 

69 self.assertEqual(obj.version, "VERSION1") 

70 self.assertEqual(set(obj.parents.all()), set()) 

71 self.assertEqual(obj.branch, None) 

72 self.assertEqual(obj.repo_type, None) 

73 self.assertEqual(obj.repo, None) 

74 self.assertEqual(obj.upstream_url, None) # Check that the invalid URL was not used 

75 self.assertEqual(obj.parameters, None) 

76 self.assertEqual(obj.build_log, None) 

77 

78 @patch('builtins.open', mock_open(read_data="Parameters\nfile")) 

79 def test_parse_run_info__parameters_file(self): 

80 self.__mock_configparser__() 

81 

82 Component.objects.create(name="COMPONENT1", description="desc", url="url", public=True) 

83 

84 cfg = """[CIRESULTS_BUILD] 

85 name: BUILD1 

86 component: COMPONENT1 

87 version: VERSION1 

88 parameters_file: file""" 

89 self.config.read_string(cfg) 

90 build = BuildResult.from_build_ini("") 

91 build.commit_to_db() 

92 

93 obj = Build.objects.get(name="BUILD1") 

94 self.assertEqual(obj.parameters, "Parameters\nfile") 

95 

96 @patch('builtins.open', mock_open(read_data="Parameters\nfile")) 

97 def test_parse_run_info__parameters_priority(self): 

98 self.__mock_configparser__() 

99 

100 Component.objects.create(name="COMPONENT1", description="desc", url="url", public=True) 

101 

102 cfg = """[CIRESULTS_BUILD] 

103 name: BUILD1 

104 component: COMPONENT1 

105 version: VERSION1 

106 parameters: parameters inline 

107 parameters_file: file""" 

108 self.config.read_string(cfg) 

109 build = BuildResult.from_build_ini("") 

110 build.commit_to_db() 

111 

112 obj = Build.objects.get(name="BUILD1") 

113 self.assertEqual(obj.parameters, "parameters inline") 

114 

115 @patch('builtins.open', mock_open(read_data="My\nBuild\nLog")) 

116 def test_parse_run_info__complete(self): 

117 self.__mock_configparser__() 

118 

119 # Create some objects in the DB that are dependencies for the new build 

120 component = Component.objects.create(name="COMPONENT1", description="desc", 

121 url="url", public=True) 

122 builds = [] 

123 for i in range(0, 2): 

124 b = Build.objects.create(name="BUILD{}".format(i), component=component, 

125 version="version") 

126 builds.append(b) 

127 

128 # Check a complete configuration 

129 cfg = """[CIRESULTS_BUILD] 

130 name: BUILD2 

131 component: COMPONENT1 

132 version: VERSION1 

133 parents: BUILD0 BUILD1 

134 branch: BRANCH1 

135 repo_type: REPO_TYPE1 

136 repo: http://repo.example 

137 upstream_url: http://URL1.de 

138 parameters: PARAMETERS WITH SPACES 

139 AND 

140 NEW 

141 LINES 

142 build_log_file: logfile 

143 """ 

144 self.config.read_string(cfg) 

145 build = BuildResult.from_build_ini("") 

146 build.commit_to_db() 

147 

148 obj = Build.objects.get(name="BUILD2") 

149 self.assertEqual(obj.name, "BUILD2") 

150 self.assertEqual(obj.component, component) 

151 self.assertEqual(obj.version, "VERSION1") 

152 self.assertEqual(set(obj.parents.all()), set(builds)) 

153 self.assertEqual(obj.branch, "BRANCH1") 

154 self.assertEqual(obj.repo_type, "REPO_TYPE1") 

155 self.assertEqual(obj.repo, "http://repo.example") 

156 self.assertEqual(obj.upstream_url, "http://URL1.de") 

157 self.assertEqual(obj.parameters, "PARAMETERS WITH SPACES\nAND\nNEW\nLINES") 

158 self.assertEqual(obj.build_log, "My\nBuild\nLog") 

159 

160 @patch('builtins.open', mock_open(read_data="My\nBuild\nLog")) 

161 def test_parse_run_info__no_ini__complete(self): 

162 # Create some objects in the DB that are dependencies for the new build 

163 component = Component.objects.create(name="COMPONENT1", description="desc", 

164 url="url", public=True) 

165 builds = [] 

166 for i in range(0, 2): 

167 b = Build.objects.create(name="BUILD{}".format(i), component=component, 

168 version="version") 

169 builds.append(b) 

170 

171 # Check a complete configuration 

172 build = BuildResult.from_dict(name="BUILD2", component="COMPONENT1", version="VERSION1", 

173 parents=["BUILD0", "BUILD1"], branch="BRANCH1", 

174 repo_type="REPO_TYPE1", repo="http://repo.example", 

175 upstream_url="http://URL1.de", 

176 parameters="PARAMETERS WITH SPACES\nAND\nNEW\nLINES", 

177 config_file="", build_log="My\nBuild\nLog") 

178 build.commit_to_db() 

179 

180 obj = Build.objects.get(name="BUILD2") 

181 self.assertEqual(obj.name, "BUILD2") 

182 self.assertEqual(obj.component, component) 

183 self.assertEqual(obj.version, "VERSION1") 

184 self.assertEqual(set(obj.parents.all()), set(builds)) 

185 self.assertEqual(obj.branch, "BRANCH1") 

186 self.assertEqual(obj.repo_type, "REPO_TYPE1") 

187 self.assertEqual(obj.repo, "http://repo.example") 

188 self.assertEqual(obj.upstream_url, "http://URL1.de") 

189 self.assertEqual(obj.parameters, "PARAMETERS WITH SPACES\nAND\nNEW\nLINES") 

190 self.assertEqual(obj.build_log, "My\nBuild\nLog") 

191 

192 def test_parse_run_info__no_ini__parents_is_None(self): 

193 # Create some objects in the DB that are dependencies for the new build 

194 component = Component.objects.create(name="COMPONENT1", description="desc", 

195 url="url", public=True) 

196 builds = [] 

197 for i in range(0, 2): 

198 b = Build.objects.create(name="BUILD{}".format(i), component=component, 

199 version="version") 

200 builds.append(b) 

201 

202 # Check a complete configuration 

203 build = BuildResult.from_dict(name="BUILD2", component="COMPONENT1", version="VERSION1", parents=[]) 

204 build.commit_to_db() 

205 

206 obj = Build.objects.get(name="BUILD2") 

207 self.assertEqual(obj.name, "BUILD2") 

208 self.assertEqual(obj.component, component) 

209 self.assertEqual(obj.version, "VERSION1") 

210 self.assertEqual(set(obj.parents.all()), set()) 

211 

212 

213class TestsuiteRunResultsTests(TestCase): 

214 def test___result_url__(self): 

215 pattern = "http://hello.world/{runconfig}/{machine}/{testsuite_build}/{run_id}/{test}" 

216 testsuite = Mock(build="build1", result_url_pattern=pattern, runconfig=Mock()) 

217 testsuite.runconfig.name = "runcfg" 

218 

219 self.assertEqual(TestsuiteRunResults.__result_url__(testsuite, 42, "machine1", "test1"), 

220 "http://hello.world/runcfg/machine1/build1/42/test1") 

221 

222 

223class TestsuiteResultsTests(TestCase): 

224 def setUp(self): 

225 self.runconfig = Mock(name="runcfg") 

226 

227 @patch('CIResults.models.Build.objects.get', return_value=Build(component=Component())) 

228 @patch('CIResults.models.TestSuite.objects.get', return_value=TestSuite()) 

229 def test_sanity(self, ts_mock, build_mock): 

230 ts_run = TestsuiteResults(self.runconfig, "name1", "build1", "piglit", 1, '') 

231 self.assertEqual(self.runconfig, self.runconfig) 

232 self.assertEqual(ts_run.name, "name1") 

233 self.assertEqual(ts_run.build, "build1") 

234 build_mock.assert_called_once() 

235 ts_mock.assert_called_once() 

236 self.assertEqual(type(ts_run.db_object), TestSuite) 

237 self.assertEqual(ts_run.format, "piglit") 

238 self.assertEqual(ts_run.version, 1) 

239 self.assertEqual(ts_run._result_type, PiglitResult) 

240 

241 @patch('CIResults.models.Build.objects.get', return_value=Build(component=Component())) 

242 @patch('CIResults.models.TestSuite.objects.get', return_value=TestSuite()) 

243 def test_invalid_format(self, ts_mock, build_mock): 

244 self.assertRaisesMessage(ValueError, "The testsuite result format 'nonexisting' is unsupported", 

245 TestsuiteResults, self.runconfig, "name1", "build1", "nonexisting", 1, '') 

246 

247 self.assertRaisesMessage(ValueError, "The version 0 of the testsuite result format 'piglit' is unsupported", 

248 TestsuiteResults, self.runconfig, "name1", "build1", "piglit", 0, '') 

249 

250 @patch('CIResults.models.Build.objects.get', return_value=Build(component=Component())) 

251 @patch('CIResults.models.TestSuite.objects.get', return_value=TestSuite()) 

252 @patch('CIResults.run_import.PiglitResult', return_value=None) 

253 def test_read_results(self, piglitresult_mock, ts_mock, build_mock): 

254 ts_run = TestsuiteResults(self.runconfig, "name1", "build1", "piglit", 1, '') 

255 ts_run.read_results("machine1", "1", "path/to/results") 

256 piglitresult_mock.assert_called_once() 

257 

258 

259class TestSuiteRunDefTests(TestCase): 

260 def test_init(self): 

261 for field in ["testsuite_build", "results_format", "results_format_version", 

262 "machine", "ts_run_id", "ts_run_path"]: 

263 kargs = {"testsuite_build": "BUILD", "results_format": "piglit", 

264 "results_format_version": 1, "machine": "MACHINE", 

265 "ts_run_id": 0, "ts_run_path": ""} 

266 kargs[field] = None 

267 

268 msg = "The parameter {} cannot be None".format(field) 

269 if field == "results_format_version" or field == "ts_run_id": 

270 msg = "The parameter {} 'None' should be an integer".format(field) 

271 

272 self.assertRaisesMessage(ValueError, msg, TestSuiteRunDef, **kargs) 

273 

274 

275class RunConfigResultsTests(TestCase): 

276 

277 def __mock_configparser__(self): 

278 # Replace the configparser instanciated by RunConfigResult with our own 

279 self.config = configparser.ConfigParser() 

280 patcher_api_call = patch('configparser.ConfigParser') 

281 mock_matches = patcher_api_call.start() 

282 mock_matches.return_value = self.config 

283 self.addCleanup(patcher_api_call.stop) 

284 patcher_api_call = patch('configparser.ConfigParser.read') 

285 mock_matches = patcher_api_call.start() 

286 mock_matches.return_value = None 

287 self.addCleanup(patcher_api_call.stop) 

288 

289 def test_parse_run_info__empty(self): 

290 self.__mock_configparser__() 

291 

292 # Check empty 

293 self.config.read_string("") 

294 self.assertRaisesMessage(ValueError, "The RunConfig file runconfig.ini is invalid", 

295 RunConfigResults, "") 

296 

297 def test_parse_run_info__name_missing(self): 

298 self.__mock_configparser__() 

299 

300 # Check name missing 

301 cfg = """[CIRESULTS_RUNCONFIG] 

302 """ 

303 self.config.read_string(cfg) 

304 self.assertRaisesMessage(ValueError, "The RunConfig file runconfig.ini is invalid: runconfig name unspecified", 

305 RunConfigResults, "") 

306 

307 @patch('os.listdir', return_value=[]) 

308 def test_parse_run_info__minimal(self, mocked_listdir): 

309 self.__mock_configparser__() 

310 

311 cfg = """[CIRESULTS_RUNCONFIG] 

312 name: RUNCFG1 

313 """ 

314 self.config.read_string(cfg) 

315 runcfg = RunConfigResults("") 

316 self.assertEqual(runcfg.name, "RUNCFG1") 

317 self.assertEqual(runcfg.url, None) 

318 self.assertEqual(runcfg.environment, None) 

319 self.assertEqual(runcfg.builds, []) 

320 self.assertEqual(runcfg.tags, []) 

321 self.assertEqual(runcfg.testsuites, {}) 

322 self.assertFalse(runcfg.temporary) 

323 

324 def test_parse_run_info__invalid_build(self): 

325 self.__mock_configparser__() 

326 

327 cfg = """[CIRESULTS_RUNCONFIG] 

328 name: RUNCFG1 

329 

330 [Testsuite1] 

331 build: build1 

332 """ 

333 self.config.read_string(cfg) 

334 msg = "The build 'build1' of the testsuite 'Testsuite1' is not found in the list of builds " 

335 msg += "of the runconfig RUNCFG1" 

336 self.assertRaisesMessage(ValueError, msg, RunConfigResults, "") 

337 

338 def test_parse_run_info__invalid_single_testsuite(self): 

339 self.__mock_configparser__() 

340 

341 cfg = """[CIRESULTS_RUNCONFIG] 

342 name: RUNCFG1 

343 builds: build1 build2 

344 

345 [CIRESULTS_TESTSUITE] 

346 build: build1 

347 

348 [Testsuite2] 

349 build: build2 

350 """ 

351 self.config.read_string(cfg) 

352 msg = "If the section CIRESULTS_TESTSUITE exists, then no additional section/testsuite can be added" 

353 self.assertRaisesMessage(ValueError, msg, RunConfigResults, "") 

354 

355 @patch('os.listdir', return_value=[]) 

356 @patch('CIResults.run_import.TestsuiteResults') 

357 def test_parse_run_info__multi_complete(self, mocked_ts_run, mocked_listdir): 

358 self.__mock_configparser__() 

359 

360 # Check a complete configuration 

361 cfg = """[CIRESULTS_RUNCONFIG] 

362 name: RUNCFG2 

363 url: http://test.url.com 

364 environment: blablabla 

365 blibliblib 

366 builds: build1 build2 build3 

367 tags: tag1 tag2 tag3 

368 temporary: true 

369 

370 [Testsuite1] 

371 build: build2 

372 format: format1 

373 result_url_pattern = pattern1 

374 

375 [Testsuite2] 

376 build: build3 

377 format: format2 

378 version: 2 

379 result_url_pattern = pattern2 

380 """ 

381 self.config.read_string(cfg) 

382 runcfg = RunConfigResults("") 

383 self.assertEqual(runcfg.name, "RUNCFG2") 

384 self.assertEqual(runcfg.url, "http://test.url.com") 

385 self.assertEqual(runcfg.environment, "blablabla\nblibliblib") 

386 self.assertEqual(runcfg.builds, ['build1', 'build2', 'build3']) 

387 self.assertEqual(runcfg.tags, ['tag1', 'tag2', 'tag3']) 

388 self.assertEqual(set(runcfg.testsuites.keys()), 

389 set(["Testsuite1", "Testsuite2"])) 

390 self.assertTrue(runcfg.temporary) 

391 

392 self.assertEqual(len(mocked_ts_run.call_args_list), 2) 

393 mocked_ts_run.assert_any_call(runcfg, 'Testsuite1', 'build2', 'format1', 1, 'pattern1') 

394 mocked_ts_run.assert_any_call(runcfg, 'Testsuite2', 'build3', 'format2', 2, 'pattern2') 

395 

396 @patch('os.listdir', return_value=[]) 

397 @patch('CIResults.run_import.TestsuiteResults') 

398 @patch('CIResults.run_import.RunConfigResults.__load_testsuite_results__') 

399 def test_parse_run_info__single_complete(self, mocked_load_results, mocked_ts_run, mocked_listdir): 

400 self.__mock_configparser__() 

401 

402 cfg = """[CIRESULTS_RUNCONFIG] 

403 name: RUNCFG2 

404 builds: build1 build2 build3 

405 tags: tag1 tag2 tag3 

406 

407 [CIRESULTS_TESTSUITE] 

408 build: build2 

409 format: format1 

410 """ 

411 self.config.read_string(cfg) 

412 runcfg = RunConfigResults("") 

413 self.assertEqual(runcfg.name, "RUNCFG2") 

414 self.assertEqual(runcfg.builds, ['build1', 'build2', 'build3']) 

415 self.assertEqual(runcfg.tags, ['tag1', 'tag2', 'tag3']) 

416 self.assertEqual(set(runcfg.testsuites.keys()), 

417 set(["CIRESULTS_TESTSUITE"])) 

418 

419 self.assertEqual(len(mocked_ts_run.call_args_list), 1) 

420 mocked_ts_run.assert_any_call(runcfg, 'CIRESULTS_TESTSUITE', 'build2', 'format1', 1, "") 

421 

422 self.assertEqual(len(mocked_load_results.call_args_list), 1) 

423 mocked_load_results.assert_any_call(runcfg._testsuites.get("CIRESULTS_TESTSUITE"), "") 

424 

425 @patch('sys.stdout', new_callable=StringIO) 

426 @patch('CIResults.run_import.TestsuiteResults') 

427 def test_load_results(self, mocked_ts_run, mocked_stdout): 

428 runcfg = RunConfigResults("CIResults/tests/results") 

429 

430 self.assertEqual(set(mocked_stdout.getvalue().splitlines()), 

431 set(["RunConfigResults: testsuite run ID 'invalid_id' should be an integer", 

432 "Ignore the testsuite 'UNREFERENCED' because it is not listed in the runconfig file"])) 

433 self.assertNotIn("UNREFERENCED", runcfg.testsuites) 

434 

435 calls = [call('machine1', 0, 'CIResults/tests/results/RENDERCHECK/machine1/0'), 

436 call('machine2', 0, 'CIResults/tests/results/RENDERCHECK/machine2/0'), 

437 call('machine1', 0, 'CIResults/tests/results/IGT/machine1/0'), 

438 call('machine1', 1, 'CIResults/tests/results/IGT/machine1/1'), 

439 call('machine2', 0, 'CIResults/tests/results/IGT/machine2/0')] 

440 mocked_ts_run.return_value.read_results.assert_has_calls(calls, any_order=True) 

441 

442 def test_init__invalid_testsuite_build(self): 

443 results = [TestSuiteRunDef(testsuite_build="INVALID", results_format="piglit", 

444 results_format_version=1, machine="machine", 

445 ts_run_id=1, ts_run_path="")] 

446 msg = "The build named 'INVALID' does not exist" 

447 self.assertRaisesMessage(ValueError, msg, RunConfigResults, name="RUNCFG", 

448 builds=['INVALID'], results=results) 

449 

450 def test_init__testsuite_build_not_in_the_list_of_builds(self): 

451 results = [TestSuiteRunDef(testsuite_build="MISSING", results_format="piglit", 

452 results_format_version=1, machine="machine", 

453 ts_run_id=1, ts_run_path="")] 

454 msg = "The build named 'MISSING' is not part of the list of builds of the runconfig" 

455 self.assertRaisesMessage(ValueError, msg, RunConfigResults, name="RUNCFG", 

456 builds=["build1", "build2"], results=results) 

457 

458 @patch('sys.stderr', new_callable=StringIO) 

459 def test_init__dual_import_of_a_testsuite_run(self, mocked_stderr): 

460 call_command('loaddata', 'CIResults/fixtures/RunConfigResults_commit_to_db', verbosity=0) 

461 

462 results = [TestSuiteRunDef(testsuite_build="build1", results_format="piglit", 

463 results_format_version=1, machine="machine", 

464 ts_run_id=1, ts_run_path="CIResults/tests/results/IGT/machine1/0/results.json.bz2"), 

465 TestSuiteRunDef(testsuite_build="build1", results_format="piglit", 

466 results_format_version=1, machine="machine", 

467 ts_run_id=1, ts_run_path="CIResults/tests/results/IGT/machine1/0/results.json.bz2")] 

468 

469 msg = "Try to import twice testsuite1's run ID 1 on the runconfig 'RUNCFG' for the machine 'machine'" 

470 self.assertRaisesMessage(ValueError, msg, RunConfigResults, name="RUNCFG", 

471 builds=["build1"], results=results) 

472 

473 @patch('os.listdir', return_value=[]) 

474 def __create_commit_to_db_env__(self, mocked_listdir, temporary=False): 

475 self.__mock_configparser__() 

476 

477 # Mock a configuration 

478 cfg = """[CIRESULTS_RUNCONFIG] 

479 name: RUNCFG 

480 url: http://test.url.com 

481 result_url_pattern = http://hello.world/{runconfig}/{machine}/{testsuite_build}/{run_id}/{test} 

482 environment: my environment 

483 builds: build1 build2 

484 tags: tag1 tag2 

485 temporary: {temporary} 

486 

487 [testsuite1] 

488 build: build1 

489 format: piglit 

490 

491 [testsuite2] 

492 build: build2 

493 format: piglit 

494 """.format(temporary=temporary, runconfig="{runconfig}", machine="{machine}", 

495 testsuite_build="{testsuite_build}", run_id="{run_id}", test="{test}") 

496 self.config.read_string(cfg) 

497 runcfg = RunConfigResults("") 

498 

499 # Add results 

500 self.start_time_run = timezone.make_aware(datetime.datetime.fromtimestamp(0), 

501 timezone.get_default_timezone()) 

502 self.start_time_test = self.start_time_run + datetime.timedelta(seconds=1) 

503 for testsuite in ["testsuite1", "testsuite2"]: 

504 for i in range(1, 3): 

505 results = [] 

506 for t in range(1, 3): 

507 results.append(TestsuiteTestResult(name="test{}".format(t), 

508 status="pass" if t != 2 else "broken", 

509 command="command", 

510 stdout="stdout", 

511 stderr="stderr", 

512 dmesg="dmesg", 

513 start_time=self.start_time_test, 

514 duration=datetime.timedelta(seconds=1))) 

515 

516 tsr = TestsuiteRunResults(testsuite=runcfg.testsuites[testsuite], 

517 machine_name="machine{}".format(i), 

518 run_id=i, test_results=results, 

519 start_time=self.start_time_run, 

520 duration=datetime.timedelta(seconds=4242)) 

521 runcfg._run_results.append(tsr) 

522 

523 return runcfg 

524 

525 @patch('sys.stdout', new_callable=StringIO) 

526 def test_commit_to_db_machine_public(self, mocked_stdout): 

527 call_command('loaddata', 'CIResults/fixtures/RunConfigResults_commit_to_db', verbosity=0) 

528 runcfg = self.__create_commit_to_db_env__() 

529 

530 # Check the amount of objects before adding anything 

531 self.assertEqual(Machine.objects.all().count(), 1) 

532 self.assertEqual(Test.objects.all().count(), 1) 

533 self.assertEqual(TextStatus.objects.all().count(), 4) 

534 self.assertEqual(TestsuiteRun.objects.all().count(), 0) 

535 self.assertEqual(TestResult.objects.all().count(), 0) 

536 self.assertEqual(KnownFailure.objects.all().count(), 0) 

537 self.assertEqual(UnknownFailure.objects.all().count(), 0) 

538 self.assertEqual(RunFilterStatistic.objects.all().count(), 0) 

539 

540 testsuite1 = TestSuite.objects.get(name="testsuite1") 

541 testsuite2 = TestSuite.objects.get(name="testsuite2") 

542 

543 # Add the runconfig to the db 

544 runcfg.commit_to_db(new_machines_public=True, new_tests_public=False) 

545 

546 # Check stdout 

547 stdout_lines = mocked_stdout.getvalue().splitlines() 

548 self.assertEqual(stdout_lines[0:-6], 

549 ['adding 1 missing machine(s)', 

550 'adding 1 missing test(s) (testsuite1)', 

551 'adding 2 missing test(s) (testsuite2)', 

552 'adding 1 missing statuse(s) (testsuite1)', 

553 'adding 1 missing statuse(s) (testsuite2)', 

554 'adding 4 testsuite runs', 

555 'adding 8 test results']) 

556 err_msg = 'Found 4 test failures (0 filters matched, 4 failures left unmatched) in ' 

557 self.assertTrue(stdout_lines[-6].startswith(err_msg), stdout_lines[-6]) 

558 

559 err_msg = 'Found 0/0 recently-archived filters matching some unknown failures in ' 

560 self.assertTrue(stdout_lines[-1].startswith(err_msg), stdout_lines[-1]) 

561 

562 # Check that the machine and test got created with the correct public attribute 

563 self.assertTrue(Machine.objects.get(name="machine2").public) 

564 self.assertFalse(Test.objects.get(name="test2", testsuite=testsuite1).public) 

565 self.assertFalse(Test.objects.get(name="test1", testsuite=testsuite2).public) 

566 self.assertFalse(Test.objects.get(name="test2", testsuite=testsuite2).public) 

567 

568 # Check that the new status is in the database for both testsuites 

569 self.assertEqual(TextStatus.objects.filter(name="broken").count(), 2) 

570 

571 # Check that the right amount of objects is found in the db 

572 self.assertEqual(Machine.objects.all().count(), 2) 

573 self.assertEqual(Test.objects.all().count(), 4) 

574 self.assertEqual(TextStatus.objects.all().count(), 6) 

575 self.assertEqual(TestsuiteRun.objects.all().count(), 4) 

576 self.assertEqual(TestResult.objects.all().count(), 8) 

577 self.assertEqual(KnownFailure.objects.all().count(), 0) 

578 self.assertEqual(UnknownFailure.objects.all().count(), 4) 

579 self.assertEqual(RunFilterStatistic.objects.all().count(), 0) 

580 

581 # Check that the testsuite of the result matches the one of the run 

582 for result in TestResult.objects.all().select_related('status', 'ts_run'): 

583 self.assertEqual(result.status.testsuite_id, result.ts_run.testsuite_id) 

584 

585 @patch('sys.stdout', new_callable=StringIO) 

586 def test_commit_to_db_test_public(self, mocked_stdout): 

587 call_command('loaddata', 'CIResults/fixtures/RunConfigResults_commit_to_db', verbosity=0) 

588 runcfg = self.__create_commit_to_db_env__() 

589 

590 testsuite1 = TestSuite.objects.get(name="testsuite1") 

591 testsuite2 = TestSuite.objects.get(name="testsuite2") 

592 

593 runcfg.commit_to_db(new_machines_public=False, new_tests_public=True) 

594 

595 # Check that the machine and test got created with the correct public attribute 

596 self.assertFalse(Machine.objects.get(name="machine2").public) 

597 self.assertTrue(Test.objects.get(name="test2", testsuite=testsuite1).public) 

598 self.assertTrue(Test.objects.get(name="test1", testsuite=testsuite2).public) 

599 self.assertTrue(Test.objects.get(name="test2", testsuite=testsuite2).public) 

600 

601 # Check that they all have the first_runconfig set correctly (new tests 

602 # got the current runconfig, and the old one still has the same value) 

603 old_runcfg = RunConfig.objects.get(name="OLD_RUNCFG") 

604 new_runcfg = RunConfig.objects.get(name="RUNCFG") 

605 for test in Test.objects.all(): 

606 runcfg = (old_runcfg if test.id == 1 else new_runcfg) 

607 self.assertEqual(test.first_runconfig, runcfg, test) 

608 

609 @patch('sys.stdout', new_callable=StringIO) 

610 def test_commit_to_db_test_temporary(self, mocked_stdout): 

611 call_command('loaddata', 'CIResults/fixtures/RunConfigResults_commit_to_db', verbosity=0) 

612 runcfg = self.__create_commit_to_db_env__(temporary=True) 

613 

614 runcfg.commit_to_db(new_machines_public=False, new_tests_public=True) 

615 

616 # Check that they all have the first_runconfig set correctly, and the new 

617 # test get None instead 

618 old_runcfg = RunConfig.objects.get(name="OLD_RUNCFG") 

619 for test in Test.objects.all(): 

620 runcfg = (old_runcfg if test.id == 1 else None) 

621 self.assertEqual(test.first_runconfig, runcfg, test) 

622 

623 def test_commit_to_db__two_builds_of_the_same_component(self): 

624 call_command('loaddata', 'CIResults/fixtures/RunConfigResults_commit_to_db', verbosity=0) 

625 

626 component1 = Component.objects.get(name="testsuite1") 

627 Build.objects.create(name="build3", component=component1) 

628 

629 runcfg = RunConfigResults(name="RUNCFG", url="http://test.url.com", 

630 result_url_pattern="", builds=["build1", "build3"], 

631 tags=["tag1", "tag2"]) 

632 

633 msg = "ERROR: Two builds (build3 and build1) cannot be from the same component (testsuite1)" 

634 self.assertRaisesMessage(ValueError, msg, runcfg.commit_to_db) 

635 

636 @patch('sys.stdout', new_callable=StringIO) 

637 def test_commit_to_db__add_one_component(self, mocked_stdout): 

638 call_command('loaddata', 'CIResults/fixtures/RunConfigResults_commit_to_db', verbosity=0) 

639 

640 # First, create a runconfig with two builds from two different testsuites 

641 runcfg = RunConfigResults(name="RUNCFG", url="http://test.url.com", 

642 result_url_pattern="", builds=["build1"], 

643 tags=["tag1", "tag2"]) 

644 runcfg.commit_to_db() 

645 

646 # Add the build2 

647 runcfg = RunConfigResults(name="RUNCFG", builds=["build2"]) 

648 runcfg.commit_to_db() 

649 

650 # Verify that the runconfig has been modified to also contain the new build 

651 runcfg_obj = RunConfig.objects.get(name="RUNCFG") 

652 self.assertEqual(set([b.id for b in runcfg_obj.builds.all()]), set([1, 2])) 

653 

654 @patch('sys.stdout', new_callable=StringIO) 

655 def test_commit_to_db__try_changing_build_of_one_component(self, mocked_stdout): 

656 call_command('loaddata', 'CIResults/fixtures/RunConfigResults_commit_to_db', verbosity=0) 

657 

658 testsuite1 = Component.objects.get(name="testsuite1") 

659 Build.objects.create(name="build3", component=testsuite1) 

660 

661 # First, create a runconfig with two builds from two different testsuites 

662 runcfg = RunConfigResults(name="RUNCFG", url="http://test.url.com", 

663 result_url_pattern="", builds=["build1", "build2"], 

664 tags=["tag1", "tag2"]) 

665 runcfg.commit_to_db() 

666 

667 runcfg = RunConfigResults(name="RUNCFG", builds=["build3"]) 

668 msg = "ERROR: Two builds (build3 and build1) cannot be from the same component (testsuite1)" 

669 self.assertRaisesMessage(ValueError, msg, runcfg.commit_to_db) 

670 

671 

672class issue_simple_stats_recomputingTests(TestCase): 

673 def setUp(self): 

674 self.filter1 = IssueFilter.objects.create(description="filter1") 

675 self.filter2 = IssueFilter.objects.create(description="filter2") 

676 self.unrelated_filter = IssueFilter.objects.create(description="unrelated filter") 

677 

678 self.issue = Issue.objects.create() 

679 self.issue.save = MagicMock() 

680 

681 IssueFilterAssociated.objects.create(filter=self.filter1, issue=self.issue) 

682 IssueFilterAssociated.objects.create(filter=self.filter2, issue=self.issue) 

683 

684 self.runconfig = RunConfig.objects.create(name='runconfig', temporary=False) 

685 

686 self.stats = dict() 

687 

688 def __check_result(self, runconfigs_covered_count=0, runconfigs_affected_count=0, 

689 last_seen_runconfig=None, save_called=False):