Coverage for CIResults/runconfigdiff.py: 73%

1from collections import namedtuple, OrderedDict, defaultdict

3from django.template.loader import render_to_string

4from django.utils.functional import cached_property

5from django.utils.safestring import mark_safe

6from django.conf import settings

8from . import models

9from .templatetags.runconfig_diff import show_suppressed

11import re

14class ExecutionTime:

15 def __init__(self, exec_time=None):

16 self.minimum = exec_time

17 self.maximum = exec_time

18 self.count = 1 if exec_time is not None else 0

20 @property

21 def is_empty(self):

22 return self.minimum is None or self.maximum is None or self.count == 0

24 def __add__(self, b):

25 if self.is_empty:

26 return b

27 elif b.is_empty:

28 return self

30 new = ExecutionTime()

31 new.minimum = min(self.minimum, b.minimum)

32 new.maximum = max(self.maximum, b.maximum)

33 new.count = self.count + b.count

35 return new

37 def __eq__(self, b):

38 return self.minimum == b.minimum and self.maximum == b.maximum and self.count == b.count

40 def __round(self, value):

41 if hasattr(value, 'total_seconds'):

42 value = value.total_seconds()

43 elif value is None:

44 return "None"

46 if int(value) == float(value):

47 return str(value)

48 else:

49 return "{:.2f}".format(value)

51 def __str__(self):

52 minimum = self.__round(self.minimum)

53 maximum = self.__round(self.maximum)

55 if maximum != minimum:

56 return "[{}, {}] s".format(minimum, maximum)

57 else:

58 return "[{}] s".format(minimum)

61class RunConfigResultsForTest:

62 def __init__(self, results):

63 self.results = []

65 if len(results) == 0:

66 raise ValueError("No results provided")

68 # Check that all results are from the same test

69 first_result_test = results[0].test

70 for result in results:

71 if result.test != first_result_test:

72 raise ValueError("Results from multiple tests")

74 # Ignore statuses that are "not run"

75 if result.status != result.status.testsuite.notrun_status:

76 self.results.append(result)

78 # Now that we checked all the results, make sure we have at least

79 # one valid result in the set

80 if len(self.results) == 0:

81 raise ValueError("No results provided")

83 @cached_property

84 def statuses(self):

85 statuses = dict()

86 for result in self.results:

87 if result.status not in statuses:

88 statuses[result.status] = []

89 statuses[result.status].append(result)

91 return statuses

93 @property

94 def exec_time(self):

95 return sum([ExecutionTime(r.duration) for r in self.results], ExecutionTime())

97 @cached_property

98 def __str(self):

99 if len(self.results) == 1:

100 return show_suppressed(self.results[0].status)

101 else:

102 # We have more than one result, which we means potentially diverging results. Check

103 # if they are diverging by creating a dictionary containing all the statuses and

104 # all their instances

105 statuses = self.statuses

106

107 if len(statuses) == 1:

108 return "( {} {} )".format(len(self.results), show_suppressed(self.results[0].status))

109 else:

110 # We have more than one status, list them all

111 status_results = []

112 for status in sorted(statuses, key=lambda b: b.name):

113 status_results.append("{} {}".format(len(statuses[status]), show_suppressed(status)))

114 return "( {} )".format(", ".join(status_results))

115

116 def __markdown_single_result(self, result):

117 return "[{}][R_{}]".format(show_suppressed(result.status), result.id)

118

119 @cached_property

120 def markdown(self):

121 if len(self.results) == 1:

122 return self.__markdown_single_result(self.results[0])

123 else:

124 results = [self.__markdown_single_result(r) for r in self.results]

125 return "({})".format(", ".join(results))

126

127 @property

128 def was_run(self):

129 return True

130

131 @cached_property

132 def failures(self):

133 failures = []

134 for result in self.results:

135 if result.is_failure:

136 failures.append(result)

137 return failures

138

139 @cached_property

140 def is_failure(self):

141 for result in self.results:

142 if result.is_failure:

143 return True

144 return False

145

146 @cached_property

147 def is_suppressed(self):

148 if len(self.failures) == 0:

149 return False

150 return all([not f.status.vetted for f in self.failures])

151

152 @cached_property

153 def associated_knownfailures(self):

154 ret = list()

155 for result in self.results:

156 ret.extend(result.known_failures_cached)

157 return ret

158

159 @cached_property

160 def all_failures_covered(self):

161 # Find the list of failures in the list of results

162 failures = set([r.id for r in self.results if r.is_failure])

163

164 # Find the list of failures that have been associated to an issue

165 known_failures = set([f.result_id for f in self.associated_knownfailures])

166

167 # If the two sets are identical, then all failures are covered by at

168 # least one known failure

169 return failures == known_failures

170

171 @cached_property

172 def issues_covering(self):

173 return set([f.matched_ifa.issue for f in self.associated_knownfailures])

174

175 @cached_property

176 def bugs_covering(self):

177 bugs = set()

178 for issue in self.issues_covering:

179 for bug in issue.bugs_cached:

180 bugs.add(bug)

181 return bugs

182

183 def __eq__(self, value):

184 return (value is not None and

185 self.statuses.keys() == value.statuses.keys() and

186 self.bugs_covering == value.bugs_covering)

187

188 def __str__(self):

189 return self.__str

190

191

192class RunConfigResultsForNotRunTest:

193 def __init__(self):

194 self.results = []

195

196 @cached_property

197 def statuses(self):

198 return dict()

199

200 @property

201 def exec_time(self):

202 return ExecutionTime()

203

204 @property

205 def __str(self):

206 return "notrun"

207

208 @property

209 def was_run(self):

210 return False

211

212 @property

213 def is_failure(self):

214 return False

215

216 @property

217 def associated_knownfailures(self):

218 return set()

219

220 @property

221 def all_failures_covered(self):

222 return set()

223

224 @property

225 def issues_covering(self):

226 return set()

227

228 @property

229 def bugs_covering(self):

230 return set()

231

232 def __eq__(self, value):

233 return value is not None and str(self) == str(value)

234

235 @property

236 def markdown(self):

237 return self.__str

238

239 def __str__(self):

240 return self.__str

241

242

243class RunConfigResultsForTestDiff:

244 # Flags

245 FIX = 1

246 REGRESSION = 2

247 WARNING = 4

248 SUPPRESSED = 8

249 KNOWN_CHANGE = 16

250 UNKNOWN_CHANGE = 32

251 NEW_TEST = 64

252

253 @property

254 def is_fix(self):

255 return (self.flags & self.FIX) > 0

256

257 @property

258 def is_regression(self):

259 return (self.flags & self.REGRESSION) > 0

260

261 @property

262 def is_warning(self):

263 return (self.flags & self.WARNING) > 0

264

265 @property

266 def is_supressed(self):

267 return (self.flags & self.SUPPRESSED) > 0

268

269 @property

270 def is_known_change(self):

271 return (self.flags & self.KNOWN_CHANGE) > 0

272

273 @property

274 def is_unknown_change(self):

275 return (self.flags & self.UNKNOWN_CHANGE) > 0

276

277 @property

278 def is_new_test(self):

279 return (self.flags & self.NEW_TEST) > 0

280

281 @property

282 def is_suppressed(self):

283 return (not self.testsuite.vetted or not self.test.vetted or

284 not self.machine.vetted or self.result_to.is_suppressed)

285

286 def __init__(self, test, testsuite, machine, result_from, result_to,

287 collapsed_differences=[]):

288 self.test = test

289 self.testsuite = testsuite

290 self.machine = machine

291 self.result_from = result_from

292 self.result_to = result_to

293 self.collapsed_differences = collapsed_differences

294

295 # Create the flags

296 self.flags = 0

297 if test.first_runconfig is None:

298 self.flags |= self.NEW_TEST

299 if result_from.is_failure and not result_to.is_failure:

300 self.flags |= self.FIX | self.KNOWN_CHANGE

301 else:

302 if result_to.is_failure and result_to.all_failures_covered:

303 self.flags |= self.KNOWN_CHANGE

304 else:

305 self.flags |= self.UNKNOWN_CHANGE

306

307 if not self.flags & self.NEW_TEST and result_to.is_failure and self.is_suppressed:

308 self.flags |= self.SUPPRESSED

309 elif (not result_from.is_failure or self.flags & self.NEW_TEST) and result_to.is_failure:

310 self.flags |= self.REGRESSION

311 else:

312 self.flags |= self.WARNING

313

314 def __issues_to_str(self, results):

315 if len(results.bugs_covering) == 0:

316 return ""

317 else:

318 bugs = set()

319 for bug in results.bugs_covering:

320 bugs.add(bug.short_name)

321 return " ([{}])".format("] / [".join(sorted(bugs)))

322

323 def __diff_to_string(self, markdown=False):

324 _from_issues = self.__issues_to_str(self.result_from)

325 _to_issues = self.__issues_to_str(self.result_to)

326

327 c_entries = ""

328 if len(self.collapsed_differences) > 0:

329 c_entries = " +{} other test{} {}".format(len(self.collapsed_differences),

330 "s" if len(self.collapsed_differences) > 1 else "",

331 self.result_to)

332

333 if markdown:

334 result_from = self.result_from.markdown

335 result_to = self.result_to.markdown

336 else:

337 result_from = str(self.result_from)

338 result_to = str(self.result_to)

339

340 ret = "{:<20}{}{} -> {}{}{}".format(show_suppressed(self.machine)+":",

341 result_from.upper(), _from_issues,

342 result_to.upper(), _to_issues,

343 c_entries)

344 if markdown:

345 for r in self.result_from.results:

346 ret += "\n [R_{}]: {}".format(r.id, r.url)

347 for r in self.result_to.results:

348 ret += "\n [R_{}]: {}".format(r.id, r.url)

349

350 return mark_safe(ret)

351

352 def __str__(self):

353 return self.__diff_to_string()

354

355 def markdown(self):

356 return self.__diff_to_string(markdown=True)

357

358

359class RunConfigResultsDiff:

360 def __init__(self, results_diff_lst, no_compress=False):

361 self._results = results_diff_lst

362 self.no_compress = no_compress

363

364 @cached_property

365 def testsuites(self):

366 testsuites = dict()

367 for diff in self._results:

368 if diff.testsuite not in testsuites:

369 testsuites[diff.testsuite] = RunConfigResultsDiff([], no_compress=self.no_compress)

370 testsuites[diff.testsuite]._results.append(diff)

371

372 ret = OrderedDict()

373 for testsuite in sorted(testsuites.keys(), key=lambda x: str(x.name)):

374 ret[testsuite] = testsuites[testsuite]

375

376 return ret

377

378 @cached_property

379 def tests(self):

380 tests = dict()

381 for diff in self._results:

382 if diff.test not in tests:

383 tests[diff.test] = RunConfigResultsDiff([], no_compress=self.no_compress)

384 tests[diff.test]._results.append(diff)

385

386 ret = OrderedDict()

387 for test in sorted(tests.keys(), key=lambda x: str(x.name)):

388 ret[test] = tests[test]

389

390 return ret

391

392 @cached_property

393 def machines(self):

394 machines = dict()

395 for diff in self._results:

396 if diff.machine not in machines:

397 machines[diff.machine] = RunConfigResultsDiff([], no_compress=self.no_compress)

398 machines[diff.machine]._results.append(diff)

399

400 ret = OrderedDict()

401 for machine in sorted(machines.keys(), key=lambda x: str(x.name)):

402 ret[machine] = machines[machine]

403

404 return ret

405

406 @cached_property

407 def to_statuses(self):

408 statuses = defaultdict(int)

409 for diff in self._results:

410 for status, results in diff.result_to.statuses.items():

411 statuses[status] += len(results)

412

413 ret = OrderedDict()

414 for status in sorted(statuses.keys(), key=lambda x: str(x.name)):

415 ret[status] = statuses[status]

416

417 return ret

418

419 @cached_property

420 def to_exec_times(self):

421 return sum([diff.result_to.exec_time for diff in self._results], ExecutionTime())

422

423 def filter(self, flags):

424 rl = list()

425 for r in self._results:

426 if (r.flags & flags) == flags:

427 rl.append(r)

428 return rl

429

430 @cached_property

431 def new_changes(self):

432 return RunConfigResultsDiff(self.filter(RunConfigResultsForTestDiff.UNKNOWN_CHANGE),

433 no_compress=self.no_compress)

434

435 @cached_property

436 def known_changes(self):

437 return RunConfigResultsDiff(self.filter(RunConfigResultsForTestDiff.KNOWN_CHANGE),

438 no_compress=self.no_compress)

439

440 @cached_property

441 def fixes(self):

442 return RunConfigResultsDiff(self.filter(RunConfigResultsForTestDiff.FIX),

443 no_compress=self.no_compress)

444

445 @cached_property

446 def regressions(self):

447 return RunConfigResultsDiff(self.filter(RunConfigResultsForTestDiff.REGRESSION),

448 no_compress=self.no_compress)

449

450 @cached_property

451 def warnings(self):

452 return RunConfigResultsDiff(self.filter(RunConfigResultsForTestDiff.WARNING),

453 no_compress=self.no_compress)

454

455 @cached_property

456 def suppressed(self):

457 return RunConfigResultsDiff(self.filter(RunConfigResultsForTestDiff.SUPPRESSED),

458 no_compress=self.no_compress)

459

460 @cached_property

461 def new_tests(self):

462 return RunConfigResultsDiff(self.filter(RunConfigResultsForTestDiff.NEW_TEST),

463 no_compress=self.no_compress)

464

465 @cached_property

466 def compressed(self):

467 if self.no_compress:

468 return self

469

470 bugs_machine = OrderedDict()

471 for r in self._results:

472 key = (r.machine, str(r.result_from), frozenset(r.result_from.bugs_covering),

473 str(r.result_to), frozenset(r.result_to.bugs_covering))

474

475 if key not in bugs_machine:

476 bugs_machine[key] = []

477 bugs_machine[key].append(r)

478

479 rl = list()

480 for bm in bugs_machine:

481 first = bugs_machine[bm][0]

482 if len(bugs_machine[bm]) > 0:

483 first = RunConfigResultsForTestDiff(first.test, first.testsuite, first.machine,

484 first.result_from, first.result_to,

485 bugs_machine[bm][1:])

486 rl.append(first)

487

488 return RunConfigResultsDiff(rl, no_compress=self.no_compress)

489

490 def __len__(self):

491 return len(self._results)

492

493 def __iter__(self):

494 return iter(self._results)

495

496

497class RunConfigResults:

498 def __init__(self, results):

499 self.results = results

500

501 @cached_property

502 def keys(self):

503 ResultKey = namedtuple('ResultKey', ['testsuite', 'test', 'machine'])

504

505 keys = set()

506 for ts in self.results:

507 for test in self.results[ts]:

508 for machine in self.results[ts][test]:

509 keys.add(ResultKey(ts, test, machine))

510 return keys

511

512 def __getitem__(self, key):

513 if (key.testsuite in self.results and

514 key.test in self.results[key.testsuite] and

515 key.machine in self.results[key.testsuite][key.test]):

516 try:

517 return RunConfigResultsForTest(self.results[key.testsuite][key.test][key.machine])

518 except ValueError:

519 # The results were invalid, say we have no results

520 return RunConfigResultsForNotRunTest()

521 else:

522 return RunConfigResultsForNotRunTest()

523

524

525class RunConfigDiff:

526 def __init__(self, runcfg_from, runcfg_to, max_missing_hosts=0.5, no_compress=False, query=None):

527 self.runcfg_from = runcfg_from

528 self.runcfg_to = runcfg_to

529 self.max_missing_hosts = max_missing_hosts

530 self.no_compress = no_compress

531 self.query = query

532

533 def __import_runcfg_results(self, **filters):

534 results = dict()

535

536 # Organise the results like this: Testsuite --> Test --> Machine --> [results]

537 failures = dict()

538 if not self.query:

539 self.query = models.TestResult

540

541 query = self.query.objects.filter(**filters).prefetch_related('test',

542 'test__first_runconfig',

543 'ts_run',

544 'ts_run__machine',

545 'ts_run__machine__aliases',

546 'status__testsuite',

547 'status').order_by('test__name')

548 for result in query.defer('start', 'duration', 'command', 'stdout', 'stderr', 'dmesg'):

549 ts = result.status.testsuite

550 if ts not in results:

551 results[ts] = OrderedDict()

552

553 if result.test not in results[ts]:

554 results[ts][result.test] = dict()

555

556 machine = result.ts_run.machine

557 if machine.aliases is not None:

558 machine = machine.aliases

559

560 if machine not in results[ts][result.test]:

561 results[ts][result.test][machine] = []

562

563 # Avoid generating a ton of requests for all the known failures by

564 # setting all results' known failures to an empty list, recording

565 # the list of failures, then later updating the results' known

566 # failures with the result of one big query for all failures

567 result.known_failures_cached = list()

568 if result.is_failure:

569 failures[result.id] = result

570

571 results[ts][result.test][machine].append(result)

572

573 # Now look for the known failures associated to the failures we found

574 query = models.KnownFailure.objects.filter(result_id__in=failures.keys())

575 for failure in query.prefetch_related('matched_ifa__issue__bugs__tracker'):

576 failures[failure.result_id].known_failures_cached.append(failure)

577

578 return results

579

580 @cached_property

581 def builds(self):

582 ret = OrderedDict()

583

584 # First, find the list of builds that differ

585 diff = self.runcfg_from.builds_ids_cached.symmetric_difference(self.runcfg_to.builds_ids_cached)

586

587 # Find from which components each differing build is from

588 builds = dict()

589 components = dict()

590 for build in models.Build.objects.filter(id__in=diff).prefetch_related('component'):

591 builds[build.id] = build

592

593 if build.component not in components:

594 components[build.component] = set([build.id])

595 else:

596 components[build.component].add(build.id)

597

598 # Go through all the components and find from/to which build it goes

599 BuildDiff = namedtuple('BuildDiff', ['from_build', 'to_build'])

600 for component in sorted(components.keys(), key=lambda c: c.name):

601 _from_ids = components[component] - self.runcfg_to.builds_ids_cached

602 _to_ids = components[component] - self.runcfg_from.builds_ids_cached

603

604 if len(_from_ids) > 1 or len(_to_ids) > 1:

605 raise ValueError("One component has multiple builds in one runconfig")

606

607 _from = builds[list(_from_ids)[0]] if len(_from_ids) == 1 else None

608 _to = builds[list(_to_ids)[0]] if len(_to_ids) == 1 else None

609

610 ret[component] = BuildDiff(_from, _to)

611

612 return ret

613

614 @cached_property

615 def builds_all(self):

616 ids = self.runcfg_from.builds_ids_cached | self.runcfg_to.builds_ids_cached

617 return sorted(models.Build.objects.filter(id__in=ids), key=lambda x: x.name)

618

619 @cached_property

620 def runcfg_from_results(self):

621 return self.__import_runcfg_results(ts_run__runconfig=self.runcfg_from)

622

623 @cached_property

624 def runcfg_to_results(self):

625 return self.__import_runcfg_results(ts_run__runconfig=self.runcfg_to)

626

627 @cached_property

628 def results(self):

629 diffs = list()

630

631 ts_from = RunConfigResults(self.runcfg_from_results)

632 ts_to = RunConfigResults(self.runcfg_to_results)

633

634 # Go through all the result keys and store the differing results

635 keys = ts_from.keys | ts_to.keys

636 for key in keys:

637 result_from = ts_from[key]

638 result_to = ts_to[key]

639 if result_to.was_run and result_from != result_to:

640 # Do not create changes for NOTRUN -> PASS

641 if not result_from.was_run and not result_to.is_failure:

642 continue

643

644 diffs.append(RunConfigResultsForTestDiff(key.test, key.testsuite, key.machine, result_from, result_to))

645

646 return RunConfigResultsDiff(diffs, no_compress=self.no_compress)

647

648 @cached_property

649 def new_tests(self):

650 diffs = list()

651

652 # no need to show anything if the to_runconfig is not temporary

653 if not self.runcfg_to.temporary:

654 return RunConfigResultsDiff(diffs)

655

656 ts_from = RunConfigResults(self.runcfg_from_results)

657 ts_to = RunConfigResults(self.runcfg_to_results)

658

659 for key in ts_to.keys:

660 result_from = ts_from[key]

661 result_to = ts_to[key]

662

663 if key.test.first_runconfig is None:

664 diffs.append(RunConfigResultsForTestDiff(key.test, key.testsuite, key.machine,

665 result_from, result_to))

666

667 return RunConfigResultsDiff(diffs, no_compress=self.no_compress)

668

669 @cached_property

670 def has_suppressed_results(self):

671 for result in self.results:

672 if result.is_suppressed:

673 return True

674 return False

675

676 @cached_property

677 def bugs(self):

678 bugs = set()

679 for result in self.results:

680 bugs.update(result.result_from.bugs_covering)

681 bugs.update(result.result_to.bugs_covering)

682 return sorted(bugs, key=lambda x: str(x.short_name))

683

684 @cached_property

685 def status(self):

686 has_regressions = False

687 has_warnings = False

688

689 for result in self.results.new_changes:

690 if not result.is_suppressed:

691 has_regressions |= result.is_regression

692 has_warnings |= result.is_warning

693

694 if has_regressions or not self.has_sufficient_machines:

695 return "FAILURE"

696 elif has_warnings:

697 return "WARNING"

698 else:

699 return "SUCCESS"

700

701 @cached_property

702 def testsuites(self):

703 TestSuiteDiff = namedtuple('TestSuiteDiff', ['all', 'runcfg_from', 'runcfg_to', 'new', 'removed'])

704

705 ts_from = self.runcfg_from_results.keys()

706 ts_to = self.runcfg_to_results.keys()

707

708 return TestSuiteDiff(all=sorted(ts_from | ts_to), runcfg_from=ts_from, runcfg_to=ts_to,

709 new=ts_to - ts_from, removed=ts_from - ts_to)

710

711 def _get_machine_list(self, results):

712 machines = set()

713 for ts in results:

714 for test in results[ts]:

715 for machine in results[ts][test]:

716 machines.add(machine)

717 return machines

718

719 @cached_property

720 def machines(self):

721 MachineDiff = namedtuple('MachineDiff', ['all', 'runcfg_from', 'runcfg_to', 'new', 'removed'])

722

723 machines_from = self._get_machine_list(self.runcfg_from_results)

724 machines_to = self._get_machine_list(self.runcfg_to_results)

725

726 return MachineDiff(all=machines_from | machines_to,

727 runcfg_from=machines_from, runcfg_to=machines_to,

728 new=machines_to - machines_from,

729 removed=machines_from - machines_to)

730

731 @property

732 def has_sufficient_machines(self):

733 return len(self.machines.removed) <= self.max_missing_hosts * len(self.machines.runcfg_from)

734

735 @cached_property

736 def text(self):

737 ret = render_to_string("CIResults/runconfigdiff.txt", {"diff": self, "bug_team_email": settings.BUG_TEAM_EMAIL})

738

739 # Look for all the [R_\d+] patterns and replace the id with a smaller one

740 matches = list(OrderedDict.fromkeys(re.findall(r'\[R_\d+\]', ret)))

741 for i, m in enumerate(matches):

742 ret = ret.replace(m, '[{}]'.format(i+1))

743

744 return ret