Coverage for CIResults/metrics.py: 70%
786 statements
« prev ^ index » next coverage.py v7.6.9, created at 2024-12-19 09:20 +0000
1from django.db.models import Sum, Max, Min, Avg
2from django.utils import timezone
3from django.utils.functional import cached_property
5from .models import Issue, BugComment, BugTracker, KnownFailure, Machine
6from .models import IssueFilterAssociated, TestsuiteRun, UnknownFailure
7from .models import TestResult, RunConfig, TextStatus, Test
9from collections import namedtuple, OrderedDict, defaultdict
10from dateutil.relativedelta import relativedelta, MO
12from datetime import timedelta
14from copy import deepcopy
16import statistics
17import dateutil
18import hashlib
19import copy
20import json
21import csv
22import io
25class Period:
26 def __init__(self, start, end, label_format='%Y-%m-%d %H:%M:%S'):
27 self.start = start
28 self.end = end
30 self.start_label = start.strftime(label_format)
31 self.end_label = end.strftime(label_format)
33 def __repr__(self):
34 return "{}->{}".format(self.start_label, self.end_label)
36 def __str__(self):
37 return repr(self)
39 def __eq__(self, other):
40 return self.start == other.start and self.end == other.end
43class Periodizer:
44 @classmethod
45 def from_json(cls, json_string):
46 params = json.loads(json_string)
48 count = int(params.get('count', 30))
50 end_date_str = params.get('end')
51 if end_date_str is not None:
52 end_date = timezone.make_aware(dateutil.parser.parse(end_date_str))
53 else:
54 end_date = timezone.now()
56 period_str = params.get('period', 'week')
57 if period_str == 'month':
58 period_offset = relativedelta(months=1, day=1, hour=0, minute=0,
59 second=0, microsecond=0)
60 period = relativedelta(months=1)
61 real_end = end_date + period_offset
62 description = 'last {} months'.format(count, )
63 if end_date_str is not None:
64 description += " before {}".format(real_end.strftime("%B %Y"))
65 label_format = "%b %Y"
66 elif period_str == 'day':
67 period_offset = relativedelta(days=1, hour=0, minute=0,
68 second=0, microsecond=0)
69 period = relativedelta(days=1)
70 real_end = end_date + period_offset
72 description = 'last {} days'.format(count)
73 if end_date_str is not None:
74 description += " before {}".format(real_end.date().isoformat())
75 label_format = "%Y-%m-%d"
76 else: # fall-through: default to week
77 period_offset = relativedelta(days=1, hour=0, minute=0,
78 second=0, microsecond=0,
79 weekday=MO(1))
80 period = relativedelta(weeks=1)
81 real_end = end_date + period_offset
82 description = 'last {} weeks'.format(count)
83 if end_date_str is not None:
84 description += " before {} / {}".format(real_end.date().isoformat(),
85 real_end.strftime("WW-%Y.%W"))
86 label_format = "WW-%Y-%W"
88 return cls(period_offset=period_offset, period=period, period_count=count,
89 end_date=end_date, description=description, label_format=label_format)
91 def __init__(self, period_offset=relativedelta(days=1, hour=0, minute=0,
92 second=0, microsecond=0,
93 weekday=MO(1)),
94 period=relativedelta(weeks=1), period_count=30,
95 end_date=timezone.now(),
96 description="last 30 weeks",
97 label_format="WW-%Y-%W"):
98 self.period_offset = period_offset
99 self.period = period
100 self.period_count = period_count
101 self.end_date = end_date
102 self.description = description
103 self.label_format = label_format
105 self.end_cur_period = end_date + period_offset
107 def __iter__(self):
108 # Reset the current position
109 self.cur_period = self.period_count
110 return self
112 def __next__(self):
113 if self.cur_period == 0:
114 raise StopIteration
116 self.cur_period -= 1
117 cur_time = self.end_cur_period - self.cur_period * self.period
118 return Period(cur_time - self.period, cur_time, self.label_format)
121PeriodOpenItem = namedtuple("PeriodOpenItem", ('period', 'label', 'active', 'new', 'closed'))
124PeriodCommentItem = namedtuple("PeriodCommentItem", ('period', 'label', 'dev_comments', 'user_comments', 'accounts'))
127class ItemCountTrend:
128 def __init__(self, items, fields=[], periodizer=None):
129 self.items = items
130 self.periodizer = periodizer
132 self.fields = defaultdict(list)
133 for i in items:
134 for field in fields:
135 values = getattr(i, field)
136 if not isinstance(values, str):
137 values = len(values)
138 self.fields[field].append(values)
140 @property
141 def stats(self):
142 r = {}
143 if self.periodizer is not None:
144 r["period_desc"] = self.periodizer.description
145 for field, values in self.fields.items():
146 r[field] = values
147 return r
150class OpenCloseCountTrend(ItemCountTrend):
151 def __init__(self, *args, **kwargs):
152 super().__init__(*args, fields=['label', 'active', 'new', 'closed'], **kwargs)
155class BugCommentCountTrend(ItemCountTrend):
156 def __init__(self, *args, **kwargs):
157 super().__init__(*args, fields=['label', 'dev_comments', 'user_comments'], **kwargs)
160def bugs_followed_since():
161 bt = BugTracker.objects.exclude(components_followed_since=None).order_by('-components_followed_since').first()
162 if bt is not None:
163 return bt.components_followed_since
164 else:
165 return None
168week_period = Periodizer.from_json('{"period": "week", "count":30}')
171# TODO: Add tests for all these functions
173def metrics_issues_over_time(user_query, periodizer=None):
174 if periodizer is None:
175 periodizer = copy.copy(week_period)
177 # Get a list of periods, from oldest to newest
178 issues = []
179 for period in periodizer:
180 issues.append(PeriodOpenItem(period, period.start_label, [], [], []))
182 if len(issues) == 0:
183 return OpenCloseCountTrend(issues, periodizer=periodizer)
185 query = user_query.objects.exclude(archived_on__lt=issues[0].period.start)
186 filtered_issues = query.exclude(added_on__gt=issues[-1].period.end)
188 for i in filtered_issues:
189 for ip in issues:
190 if i.added_on >= ip.period.start and i.added_on < ip.period.end:
191 ip.new.append(i)
192 elif i.archived_on is not None and i.archived_on >= ip.period.start and i.archived_on < ip.period.end:
193 ip.closed.append(i)
194 elif i.added_on < ip.period.start and (i.archived_on is None or i.archived_on >= ip.period.end):
195 ip.active.append(i)
197 return OpenCloseCountTrend(issues, periodizer=periodizer)
200def metrics_bugs_over_time(user_query, periodizer=None):
201 if periodizer is None:
202 periodizer = copy.copy(week_period)
204 # Find out what is the earliest components_followed_since and make sure we don't go that far
205 earliest_followed_since = bugs_followed_since()
206 if earliest_followed_since is None:
207 return (OpenCloseCountTrend([], periodizer=periodizer), BugCommentCountTrend([], periodizer=periodizer))
209 # Get a list of periods, from oldest to newest
210 bug_periods = []
211 comment_periods = []
212 for period in periodizer:
213 # Ignore all the periods not fully covered
214 if period.start < earliest_followed_since:
215 continue
216 bug_periods.append(PeriodOpenItem(period, period.start_label, [], [], []))
217 comment_periods.append(PeriodCommentItem(period, period.start_label, [], [], None))
219 followed_bug_ids = set()
220 filtered_bugs = user_query.objects.exclude(closed__lt=earliest_followed_since)
221 for bug in filtered_bugs.prefetch_related("tracker"):
222 # ignore bugs that we are not following
223 if bug.tracker.tracker.has_components and bug.component not in bug.tracker.components_followed_list:
224 continue
226 # ignore bugs that do not have a created / closed value yet
227 if not bug.created or not bug.closed:
228 continue
230 for bp in bug_periods:
231 if bug.created >= bp.period.start and bug.created < bp.period.end:
232 bp.new.append(bug)
233 elif bug.closed is not None and bug.closed >= bp.period.start and bug.closed < bp.period.end:
234 bp.closed.append(bug)
235 elif bug.created < bp.period.start and (bug.closed is None or bug.closed >= bp.period.end):
236 bp.active.append(bug)
238 # Keep track of all the bugs we used for the open/close count
239 followed_bug_ids.add(bug.id)
241 # Fetch all comments made on the followed bugs and order them in chronological
242 # order, before associating them to their corresponding bug period
243 idx = 0
244 followed_comments = BugComment.objects.filter(bug_id__in=followed_bug_ids,
245 created_on__gte=comment_periods[0].period.start,
246 created_on__lte=periodizer.end_cur_period).order_by('created_on')
247 for comment in followed_comments.prefetch_related('account'):
248 cp = comment_periods[idx]
250 # go to the next period if the comment has been made after the end of the current period
251 while comment.created_on > cp.period.end:
252 idx += 1
253 cp = comment_periods[idx]
255 if comment.account.is_developer:
256 cp.dev_comments.append(comment)
257 else:
258 cp.user_comments.append(comment)
259 return (OpenCloseCountTrend(bug_periods, periodizer=periodizer),
260 BugCommentCountTrend(comment_periods, periodizer=periodizer))
263def metrics_comments_over_time(user_query, periodizer=None):
264 if periodizer is None:
265 periodizer = Periodizer.from_json('{"period": "week", "count": 8}')
267 earliest_followed_since = bugs_followed_since()
268 if earliest_followed_since is None:
269 return BugCommentCountTrend([], periodizer=periodizer), {}
271 # Get a list of periods, from oldest to newest
272 comment_periods = []
273 for period in periodizer:
274 # Ignore all the periods not fully covered
275 if period.start < earliest_followed_since:
276 continue
277 period = PeriodCommentItem(period, period.start_label, [], [], defaultdict(list))
278 comment_periods.append(period)
280 if len(comment_periods) == 0:
281 return BugCommentCountTrend([], periodizer=periodizer), {}
283 # Get the list of comments wanted by the user, excluding comments made before the moment we started polling
284 query = user_query.objects.exclude(created_on__lt=comment_periods[0].period.start)
285 filtered_comments = query.exclude(created_on__gt=comment_periods[-1].period.end)
287 idx = 0
288 cp = comment_periods[idx]
289 accounts_found = set()
290 for comment in filtered_comments.prefetch_related('bug__tracker', 'account__person').order_by('created_on'):
291 # ignore comments on bugs that we are not following
292 if comment.bug.tracker.tracker.has_components:
293 if comment.bug.component not in comment.bug.tracker.components_followed_list:
294 continue
296 # go to the next period if the comment has been made after the end of the current period
297 while comment.created_on > cp.period.end:
298 idx += 1
299 cp = comment_periods[idx]
301 # Add the comment to the global counter
302 if comment.account.is_developer:
303 cp.dev_comments.append(comment)
304 else:
305 cp.user_comments.append(comment)
307 # Add the comment to the per-account list
308 accounts_found.add(comment.account)
309 cp.accounts[comment.account].append(comment)
311 # Create an dictionnary that keeps track of comments per account
312 sorted_account = sorted(accounts_found, key=lambda a: str(a))
313 per_account_periods = OrderedDict()
314 for account in sorted_account:
315 per_account_periods[account] = []
316 for cp in comment_periods:
317 per_account_periods[account].append(cp.accounts[account])
319 return BugCommentCountTrend(comment_periods, periodizer=periodizer), per_account_periods
322class Bin:
323 def __init__(self, upper_limit, label):
324 self.items = set()
325 self.upper_limit = upper_limit
326 self.label = label
329class TimeBinizer:
330 def __init__(self, items, bins=[Bin(timedelta(hours=1), "under an hour"),
331 Bin(timedelta(hours=6), "under 6 hours"),
332 Bin(timedelta(days=1), "under a day"),
333 Bin(timedelta(days=7), "under a week"),
334 Bin(timedelta(days=30), "under a month"),
335 Bin(timedelta(days=90), "under 3 months"),
336 Bin(timedelta(days=365), "under a year"),
337 Bin(timedelta.max, "over a year")]):
338 self._bins = deepcopy(bins)
339 for item, time in items:
340 for bin in self._bins:
341 if time < bin.upper_limit:
342 bin.items.add(item)
343 break
345 @property
346 def bins(self):
347 return self._bins
349 @property
350 def stats(self):
351 return {"items_count": [len(b.items) for b in self.bins],
352 "label": [b.label for b in self.bins]}
355def metrics_issues_ttr(date=timezone.now(), period=timedelta(days=30)):
356 request = Issue.objects.filter(archived_on__lt=date, archived_on__gt=date-period).exclude(archived_on__isnull=True)
357 return TimeBinizer([(item, item.archived_on - item.added_on)
358 for item in request if item.archived_on and item.added_on])
361def metrics_open_issues_age(date=timezone.now()):
362 request = Issue.objects.filter(archived_on=None)
363 now = timezone.now()
364 return TimeBinizer([(item, now - item.added_on) for item in request if item.added_on])
367def metrics_failure_filing_delay(date=timezone.now(), period=timedelta(days=30)):
368 request = KnownFailure.objects.filter(manually_associated_on__gt=date-period)
369 request = request.filter(result__ts_run__runconfig__temporary=False)
370 bins = [Bin(timedelta(hours=1), "under an hour"),
371 Bin(timedelta(hours=8), "under 8 hours"),
372 Bin(timedelta(days=1), "under a day"),
373 Bin(timedelta(days=3), "under three days"),
374 Bin(timedelta(days=7), "under a week"),
375 Bin(timedelta(days=30), "under a month"),
376 Bin(timedelta.max, "over a month")]
377 return TimeBinizer([(item, item.filing_delay) for item in request], bins=bins)
380def metrics_bugs_ttr(user_query, date=timezone.now(), period=timedelta(days=30)):
381 request = user_query.objects.filter(closed__lt=date, closed__gt=date-period)
382 request = request.exclude(closed__isnull=True).prefetch_related("tracker")
383 bugs = set()
384 for bug in request:
385 if bug.tracker.tracker.has_components:
386 if bug.component in bug.tracker.components_followed_list:
387 bugs.add(bug)
388 else:
389 bugs.add(bug)
390 return TimeBinizer([(item, item.closed - item.created) for item in bugs if item.closed and item.created])
393def metrics_open_bugs_age(user_query, date=timezone.now()):
394 request = user_query.objects.filter(closed=None).prefetch_related("tracker")
395 bugs = set()
396 for bug in request:
397 if bug.tracker.tracker.has_components:
398 if bug.component in bug.tracker.components_followed_list:
399 bugs.add(bug)
400 else:
401 bugs.add(bug)
402 now = timezone.now()
403 return TimeBinizer([(item, now - item.created) for item in bugs if item.created])
406class PieChartData:
407 # results needs to be a dictionary mapping a label (string) to a number
408 def __init__(self, results, colors=dict()):
409 self._results = OrderedDict()
410 self._colors = colors
412 for label, value in sorted(results.items(), key=lambda kv: kv[1], reverse=True):
413 self._results[label] = value
415 def label_to_color(self, label):
416 color = self._colors.get(label)
418 if color is not None:
419 return color
420 else:
421 blake2 = hashlib.blake2b()
422 blake2.update(label.encode())
423 return "#" + blake2.hexdigest()[-7:-1]
425 @property
426 def colors(self):
427 return [self.label_to_color(label) for label in self._results.keys()]
429 def stats(self):
430 return {'results': list(self._results.values()),
431 'labels': list(self._results.keys()),
432 'colors': list(self.colors)}
435class ColouredObjectPieChartData(PieChartData):
436 def __init__(self, objects):
437 results = defaultdict(int)
438 for obj in objects:
439 results[str(obj)] += 1
441 colors = {}
442 for obj in set(objects):
443 colors[str(obj)] = obj.color
445 super().__init__(results, colors)
448def metrics_testresult_statuses_stats(results):
449 return ColouredObjectPieChartData([r.status for r in results])
452def metrics_knownfailure_statuses_stats(failures):
453 return ColouredObjectPieChartData([f.result.status for f in failures])
456def metrics_testresult_machines_stats(results):
457 return ColouredObjectPieChartData([r.ts_run.machine for r in results])
460def metrics_knownfailure_machines_stats(failures):
461 return ColouredObjectPieChartData([f.result.ts_run.machine for f in failures])
464def metrics_testresult_tests_stats(results):
465 tests = {}
466 for result in results:
467 label = str(result.test)
468 tests[label] = tests.get(label, 0) + 1
469 return PieChartData(tests)
472def metrics_knownfailure_tests_stats(failures):
473 return metrics_testresult_tests_stats([f.result for f in failures])
476def metrics_knownfailure_issues_stats(failures):
477 # Since the query has likely already been made, we can't prefetch the
478 # issues anymore... so let's hand roll it!
479 matched_ifas = set()
480 for failure in failures:
481 matched_ifas.add(failure.matched_ifa_id)
482 ifa_to_issues = dict()
483 ifas = IssueFilterAssociated.objects.filter(id__in=matched_ifas)
484 for e in ifas.prefetch_related('issue', 'issue__bugs', 'issue__bugs__tracker'):
485 ifa_to_issues[e.id] = e.issue
487 issues = {}
488 for failure in failures:
489 label = str(ifa_to_issues.get(failure.matched_ifa_id, None))
490 issues[label] = issues.get(label, 0) + 1
491 return PieChartData(issues)
494def metrics_testresult_issues_stats(failures):
495 total = []
496 for failure in failures:
497 total.extend(failure.known_failures.all())
498 return metrics_knownfailure_issues_stats(total)
501class Rate:
502 def __init__(self, count, total):
503 self.count = count
504 self.total = total
506 @property
507 def percent(self):
508 if self.total > 0:
509 return self.count / self.total * 100.0
510 else:
511 return 0.0
513 def __repr__(self):
514 return "Rate({}, {})".format(self.count, self.total)
516 def __str__(self):
517 return "{:.2f}% ({} / {})".format(self.percent, self.count, self.total)
520class Statistics:
521 def __init__(self, unit, samples=None):
522 self.unit = unit
524 if samples is None:
525 self.samples = []
526 else:
527 self.samples = samples
529 def add(self, sample):
530 self.samples.append(sample)
532 def __iadd__(self, sample):
533 self.add(sample)
534 return self
536 @property
537 def min(self):
538 return min(self.samples)
540 @property
541 def max(self):
542 return max(self.samples)
544 @property
545 def mean(self):
546 return statistics.mean(self.samples)
548 @property
549 def median(self):
550 return statistics.median(self.samples)
552 @property
553 def stdev(self):
554 return statistics.stdev(self.samples) if len(self.samples) > 1 else 0
556 def __str__(self):
557 mean = self.mean
558 if mean > 0:
559 stdev = self.stdev
560 return "{:.3f} {} ±{:.1f}%".format(mean, self.unit, stdev / mean * 100.0)
561 else:
562 return "0 {}".format(self.unit)
565class LineChartData:
566 # results needs to be a dictionary mapping a label (string) to a number
567 def __init__(self, results, x_labels, line_label_colors={}):
568 self._results = OrderedDict()
569 self.x_labels = x_labels
570 self.line_label_colors = line_label_colors
572 for label, value in results.items():
573 self._results[label] = value
575 def label_to_color(self, label):
576 blake2 = hashlib.blake2b()
577 blake2.update(label.encode())
578 default = "#" + blake2.hexdigest()[-7:-1]
580 return self.line_label_colors.get(label, default)
582 def stats(self):
583 dataset = []
584 for line_label, result in self._results.items():
585 color = self.label_to_color(line_label)
586 entry = {'label': line_label,
587 'borderColor': color,
588 'backgroundColor': color,
589 'data': result}
590 dataset.append(entry)
592 return {'dataset': dataset, 'labels': self.x_labels}
595class MetricPassRatePerRunconfig:
596 filtering_model = TestResult
598 def _queryset_to_dict(self, Model, ids, *prefetch_related):
599 return dict((o.pk, o) for o in Model.objects.filter(id__in=ids).prefetch_related(*prefetch_related))
601 def __init__(self, user_query):
602 self.query = user_query
603 db_results = self.query.objects.values_list('id', 'status_id', 'ts_run__runconfig_id')
605 # Collect all the failures
606 statuses = self._queryset_to_dict(TextStatus, [r[1] for r in db_results], 'testsuite')
607 failure_status_ids = set([s.id for s in statuses.values() if s.is_failure])
608 failures = set()
609 for result_id, status_id, runconfig_id in db_results:
610 if status_id in failure_status_ids:
611 failures.add(result_id)
612 del failure_status_ids
614 # Find the related issues
615 known_failures = KnownFailure.objects.filter(result__in=failures).values_list('result_id',
616 'matched_ifa__issue_id',
617 'matched_ifa__issue__expected')
618 issues = self._queryset_to_dict(Issue, [r[1] for r in known_failures],
619 'bugs__tracker', 'bugs__assignee__person')
620 self._issue_hit_count = defaultdict(int) # [issue] = int
621 expected_failures = set() # [failure_ids](issues)
622 for result_id, issue_id, issue_expected in known_failures:
623 issue = issues[issue_id]
624 if issue_expected:
625 expected_failures.add(result_id)
626 else:
627 self._issue_hit_count[issue] += 1
628 self.total_result_count = len(db_results)
629 self.kept_result_count = self.total_result_count - len(expected_failures)
630 del known_failures
632 # Count the results per runconfig and statuses
633 runconfigs = self._queryset_to_dict(RunConfig, [r[2] for r in db_results])
634 runconfigs_tmp = defaultdict(lambda: defaultdict(int))
635 statuses_tmp = set()
636 for result_id, status_id, runconfig_id in db_results:
637 if result_id not in expected_failures:
638 status = statuses[status_id]
639 runconfigs_tmp[runconfigs[runconfig_id]][status] += 1
640 statuses_tmp.add(status)
641 del expected_failures
643 # Order the statuses and runconfigs
644 runconfigs_ordered = sorted(runconfigs_tmp.keys(), key=lambda r: r.added_on, reverse=True)
645 statuses_ordered = sorted(statuses_tmp, key=lambda r: str(r))
647 # Create the final result structures
648 self.runconfigs = OrderedDict() # [runconfig][status] = Rate()
649 self.statuses = OrderedDict() # [status][runconfig] = Rate()
650 self.results_count = dict() # [runconfig] = int
651 for status in statuses_ordered:
652 self.statuses[status] = OrderedDict()
653 for runconfig in runconfigs_ordered:
654 # Compute the total for the run
655 total = 0
656 for status in statuses_ordered:
657 total += runconfigs_tmp[runconfig][status]
658 self.results_count[runconfig] = total
660 # Add the passrate to both the runconfig-major and status-major table
661 self.runconfigs[runconfig] = OrderedDict()
662 for status in statuses_ordered:
663 passrate = Rate(runconfigs_tmp[runconfig][status], total)
664 self.runconfigs[runconfig][status] = self.statuses[status][runconfig] = passrate
666 @property
667 def discarded_rate(self):
668 return Rate(self.total_result_count - self.kept_result_count, self.total_result_count)
670 @cached_property
671 def chart(self):
672 # The runconfigs are ordered from newest to oldest, reverse that
673 runconfigs = list(reversed(self.runconfigs.keys()))
675 chart_data = defaultdict(list)
676 for runconfig in runconfigs:
677 for status in self.runconfigs[runconfig]:
678 passrate = self.runconfigs[runconfig][status]
679 chart_data[str(status)].append(format((passrate.count / passrate.total) * 100, '.3f'))
681 status_colors = dict()
682 for status in self.statuses:
683 status_colors[str(status)] = status.color
685 return LineChartData(chart_data, [r.name for r in runconfigs], status_colors)
687 @cached_property
688 def to_csv(self):
689 f = io.StringIO()
690 writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
692 writer.writerow(['Run Config', 'Results count'] + [str(s) for s in self.statuses])
693 for runconfig, statuses in self.runconfigs.items():
694 writer.writerow([runconfig, self.results_count[runconfig]] + [p.count for status, p in statuses.items()])
696 return f.getvalue()
698 @cached_property
699 def most_hit_issues(self):
700 ordered_issues = OrderedDict()
701 for issue in sorted(self._issue_hit_count, key=lambda i: self._issue_hit_count[i], reverse=True):
702 occ_count = self._issue_hit_count[issue]
703 ordered_issues[issue] = Rate(occ_count, self.kept_result_count)
704 return ordered_issues
707class MetricPassRatePerTest:
708 filtering_model = TestResult
710 discarded_label = 'discarded (expected)'
711 discarded_color = '#cc99ff'
713 class AggregatedTestResults:
714 def __init__(self, test):
715 self.test = test
717 self.results_count = 0
718 self.statuses = defaultdict(int)
719 self.duration = Statistics('s')
720 self.issues = defaultdict(int)
721 self.overall_result = self.test.testsuite.notrun_status
722 self.worst_failures = []
724 self.discarded_results_count = 0
725 self.discarded_count_per_issue = defaultdict(int) # [issue] = count
727 def add_result(self, result_id, status, duration, issues=None):
728 self.results_count += 1
729 self.statuses[status] += 1
730 self.duration.add(duration.total_seconds())
731 if issues is not None:
732 for issue in issues:
733 self.issues[issue] += 1
735 if self.overall_result is None or status.actual_severity > self.overall_result.actual_severity:
736 self.overall_result = status
737 self.worst_failures = []
739 if status == self.overall_result:
740 self.worst_failures.append(result_id)
742 def add_discarded_result(self, result_id, duration, issues=None):
743 self.discarded_results_count += 1
744 self.duration.add(duration.total_seconds())
745 for issue in issues:
746 self.discarded_count_per_issue[issue] += 1
748 def status_occurence_rate(self, status):
749 return Rate(self.statuses.get(status, 0), self.results_count)
751 def issue_occurence_rate(self, issue):
752 return Rate(self.issues.get(issue, 0), self.results_count)
754 @property
755 def rate_of_worst_failure(self):
756 return self.status_occurence_rate(self.overall_result)
758 @property
759 def is_fully_discarded(self):
760 return self.results_count == 0 and self.discarded_results_count > 0
762 @property
763 def issue_occurence_rates(self):
764 rates = dict()
765 if self.is_fully_discarded:
766 for issue in self.discarded_count_per_issue:
767 rates[issue] = Rate(self.discarded_count_per_issue.get(issue, 0),
768 self.results_count + self.discarded_results_count)
769 else:
770 for issue in self.issues:
771 rates[issue] = self.issue_occurence_rate(issue)
772 return rates
774 @cached_property
775 def is_pass(self):
776 # Notruns are not pass
777 if self.overall_result.is_notrun:
778 return False
780 return not self.overall_result.is_failure
782 PassRateStatistics = namedtuple('PassRateStatistics', 'passrate runrate discarded_rate notrun_rate')
784 def _queryset_to_dict(self, Model, ids, *prefetch_related):
785 return dict((o.pk, o) for o in Model.objects.filter(id__in=ids).prefetch_related(*prefetch_related))
787 def __init__(self, user_query):
788 self.query = user_query
790 # Create the final result structures
791 self.tests = OrderedDict() # [test] = AggregatedTestResults()
792 self.statuses = OrderedDict() # [status] = Rate()
794 # Fetch the user-filtered from the database
795 db_results = self.query.objects.values_list('id', 'test_id', 'status_id', 'ts_run_id', 'duration')
797 # This query will lazily be evaluated, so it can be set early
798 ts_runs = TestsuiteRun.objects.filter(id__in=[r[3] for r in db_results]).values_list('machine_id',
799 'runconfig_id')
801 # Collect all the failures and ts_run_ids that got some notruns
802 statuses = self._queryset_to_dict(TextStatus, [r[2] for r in db_results], 'testsuite')
803 failure_status_ids = set([s.id for s in statuses.values() if s.is_failure])
804 notrun_status_ids = set([s.id for s in statuses.values() if s.is_notrun])
805 interrupted_tsruns_notruns_count = defaultdict(int)
806 notrun_count = 0
807 failure_ids = set()
808 for result_id, test_id, status_id, ts_run_id, duration in db_results:
809 if status_id in failure_status_ids:
810 failure_ids.add(result_id)
811 elif status_id in notrun_status_ids:
812 interrupted_tsruns_notruns_count[ts_run_id] += 1
813 notrun_count += 1
814 del failure_status_ids
816 # Find the last test of every interrupted ts_run
817 q = TestResult.objects.filter(ts_run__in=interrupted_tsruns_notruns_count.keys())
818 q = q.exclude(status_id__in=notrun_status_ids).values_list('ts_run__id').annotate(last_test_run_id=Max('id'))
819 last_result_of_tsr = set([v[1] for v in q])
820 q2 = KnownFailure.objects.filter(result_id__in=last_result_of_tsr).values_list('result__ts_run_id',
821 'matched_ifa__issue_id')
822 run_to_issue = dict(q2)
824 id_to_issue = dict([(i.id, i) for i in Issue.objects.filter(id__in=run_to_issue.values())])
825 issue_not_run_impact = defaultdict(int)
826 for run_id, issue_id in run_to_issue.items():
827 issue = id_to_issue[issue_id]
828 notruns_count = interrupted_tsruns_notruns_count.pop(run_id)
829 issue_not_run_impact[issue] += notruns_count
831 undocumented_failure_not_run_count = 0
832 q3 = UnknownFailure.objects.filter(result_id__in=last_result_of_tsr).values_list('result__ts_run_id', flat=True)
833 for undocumented_interrupted_ts_run_id in set(q3):
834 notruns_count = interrupted_tsruns_notruns_count.pop(undocumented_interrupted_ts_run_id)
835 undocumented_failure_not_run_count += notruns_count
837 unexplained_notruns_count = sum(interrupted_tsruns_notruns_count.values())
838 del run_to_issue
839 del id_to_issue
840 del interrupted_tsruns_notruns_count
842 # Find the related issues
843 known_failures = KnownFailure.objects.filter(result__in=failure_ids).values_list('result_id',
844 'matched_ifa__issue_id',
845 'matched_ifa__issue__expected')
846 issues = self._queryset_to_dict(Issue, [r[1] for r in known_failures],
847 'bugs__tracker', 'bugs__assignee__person')
848 issue_hit_count = defaultdict(int) # [issue] = Rate
849 failures = defaultdict(set) # [result_id] = set(issues)
850 expected_failures = defaultdict(set) # [failure_ids](issues)
851 explained_failure_ids = set()
852 for result_id, issue_id, issue_expected in known_failures:
853 issue = issues[issue_id]
854 if issue_expected:
855 expected_failures[result_id].add(issue)
856 else:
857 failures[result_id].add(issue)
858 issue_hit_count[issue] += 1
859 explained_failure_ids.add(result_id)
860 self.total_result_count = len(db_results)
861 self.kept_result_count = self.total_result_count - len(expected_failures)
862 self.uncovered_failure_rate = Rate(len(failure_ids - explained_failure_ids), len(failure_ids))
863 self.notrun_rate = Rate(notrun_count, self.total_result_count)
864 del known_failures
865 del failure_ids
866 del explained_failure_ids
868 # Create all the aggregated results for each test, in the alphabetical order
869 tests = self._queryset_to_dict(Test, [r[1] for r in db_results], 'testsuite')
870 for test in sorted(tests.values(), key=lambda t: str(t)):
871 self.tests[test] = self.AggregatedTestResults(test)
873 # Move all the results to their right category
874 status_result_count = defaultdict(int)
875 for result_id, test_id, status_id, ts_run_id, duration in db_results:
876 test = tests[test_id]
878 if result_id not in expected_failures:
879 status = statuses[status_id]
880 status_result_count[status] += 1
881 self.tests[test].add_result(result_id, status, duration, failures.get(result_id))
882 else:
883 self.tests[test].add_discarded_result(result_id, duration, expected_failures.get(result_id))
885 # We do not need these results anymore, so free them
886 del tests
887 del failures
888 del expected_failures
889 del issues
890 del db_results
892 # Get the list of machines and runconfigs
893 self.machines = Machine.objects.filter(id__in=[r[0] for r in ts_runs])
894 self.runconfigs = RunConfig.objects.filter(id__in=[r[1] for r in ts_runs])
896 # Compute the results-level stats
897 self.most_hit_issues = OrderedDict()
898 for issue in sorted(issue_hit_count.keys(), key=lambda i: issue_hit_count[i], reverse=True): # noqa
899 self.most_hit_issues[issue] = Rate(issue_hit_count[issue], self.kept_result_count)
900 del issue_hit_count
902 self.most_interrupting_issues = OrderedDict()
903 self.explained_interruption_rate = Rate(0, self.notrun_rate.count)
904 for issue in sorted(issue_not_run_impact.keys(), key=lambda i: issue_not_run_impact[i], reverse=True): # noqa
905 rate = Rate(issue_not_run_impact[issue], self.notrun_rate.count)
906 self.most_interrupting_issues[issue] = rate
907 self.explained_interruption_rate.count += rate.count
908 self.unknown_failure_interruption_rate = Rate(undocumented_failure_not_run_count, self.notrun_rate.count)
909 self.unexplained_interruption_rate = Rate(unexplained_notruns_count, self.notrun_rate.count)
911 # Create all the status rates in alphabetical order and compute their occurence rate
912 statuses_ordered = sorted(status_result_count.keys(), key=lambda r: str(r),
913 reverse=True) # only the statuses we kept
914 for status in statuses_ordered:
915 self.statuses[status] = Rate(status_result_count[status], self.kept_result_count)
917 @cached_property
918 def result_statuses_chart(self):
919 statuses = {}
920 colors = {}
921 for status, rate in self.statuses.items():
922 label = str(status)
923 statuses[label] = rate.count
924 colors[label] = status.color
926 discarded_count = self.total_result_count - self.kept_result_count
927 if discarded_count > 0:
928 statuses[self.discarded_label] = discarded_count
929 colors[self.discarded_label] = self.discarded_color
931 return PieChartData(statuses, colors)
933 @cached_property
934 def uncovered_failure_rate_chart(self):
935 statuses = {
936 'filed / documented': self.uncovered_failure_rate.total - self.uncovered_failure_rate.count,
937 'need filing': self.uncovered_failure_rate.count,
938 }
940 colors = {
941 'filed / documented': '#33cc33',
942 'need filing': '#ff0000',
943 }
945 return PieChartData(statuses, colors)
947 @cached_property
948 def aggregated_statuses_chart(self):
949 statuses = defaultdict(int)
950 colors = {}
952 for test, agg_result in self.tests.items():
953 if agg_result.is_fully_discarded:
954 label = self.discarded_label
955 else:
956 label = str(agg_result.overall_result)
957 statuses[label] += 1
959 colors[self.discarded_label] = self.discarded_color
960 for status in self.statuses:
961 colors[str(status)] = status.color
963 return PieChartData(statuses, colors)
965 @cached_property
966 def passrate_chart(self):
967 stats = self.statistics
969 statuses = {
970 'pass rate': stats.passrate.count,
971 'fail rate': stats.runrate.count - stats.passrate.count,
972 'discarded rate': stats.discarded_rate.count,
973 'notrun rate': stats.notrun_rate.count,
974 }
976 colors = {
977 'pass rate': '#33cc33',
978 'fail rate': '#ff0000',
979 'discarded rate': self.discarded_color,
980 'notrun rate': '#000000',
981 }
983 return PieChartData(statuses, colors)
985 @cached_property
986 def raw_statistics(self):
987 passrate = Rate(0, self.total_result_count)
988 runrate = Rate(0, self.total_result_count)
989 discarded_rate = Rate(0, self.total_result_count)
990 notrun_rate = Rate(0, self.total_result_count)
992 self.kept_result_count = self.total_result_count
994 for test, results in self.tests.items():
995 if results.is_fully_discarded:
996 discarded_rate.count += 1
997 elif not results.overall_result.is_notrun:
998 runrate.count += 1
999 if not results.overall_result.is_failure:
1000 passrate.count += 1
1001 else:
1002 notrun_rate.count += 1
1004 return self.PassRateStatistics(passrate, runrate, discarded_rate, notrun_rate)
1006 @cached_property
1007 def statistics(self):
1008 passrate = Rate(0, len(self.tests))
1009 runrate = Rate(0, len(self.tests))
1010 discarded_rate = Rate(0, len(self.tests))
1011 notrun_rate = Rate(0, len(self.tests))
1013 for test, results in self.tests.items():
1014 if results.is_fully_discarded:
1015 discarded_rate.count += 1
1016 elif not results.overall_result.is_notrun:
1017 runrate.count += 1
1018 if not results.overall_result.is_failure:
1019 passrate.count += 1
1020 else:
1021 notrun_rate.count += 1
1023 return self.PassRateStatistics(passrate, runrate, discarded_rate, notrun_rate)
1025 @cached_property
1026 def total_execution_time(self):
1027 total = 0
1028 for results in self.tests.values():
1029 total += results.duration.mean
1030 return timedelta(seconds=int(total))
1033class MetricRuntimeHistory:
1034 filtering_model = TestsuiteRun
1036 def _queryset_to_dict(self, Model, ids, *prefetch_related):
1037 return dict((o.pk, o) for o in Model.objects.filter(id__in=ids).prefetch_related(*prefetch_related))
1039 def __init__(self, user_query, average_per_machine=False):
1040 self.query = user_query
1041 self.average_per_machine = average_per_machine
1043 # Fetch the user-filtered from the database
1044 filtered_results = self.query.objects
1045 filtered_results = filtered_results.filter(duration__gt=timedelta(seconds=0)) # Ignore negative exec times
1046 db_results = filtered_results.values('id', 'runconfig_id',
1047 'machine_id').annotate(total=Sum("duration"))
1049 # Get the runconfigs and machines
1050 runconfigs = self._queryset_to_dict(RunConfig, [r['runconfig_id'] for r in db_results])
1051 machines = self._queryset_to_dict(Machine, [r['machine_id'] for r in db_results], "aliases")
1052 self._tsr_ids = [r['id'] for r in db_results]
1054 # Add the results to the history, then compute the totals
1055 self.participating_machines = defaultdict(lambda: defaultdict(set))
1056 runconfigs_tmp = defaultdict(lambda: defaultdict(timedelta))
1057 machines_tmp = set()
1058 for r in db_results:
1059 # de-duplicate shard machines by replacing a machine by its shard
1060 machine = machines[r['machine_id']]
1061 if machine.aliases is not None:
1062 machine = machine.aliases
1064 runconfigs_tmp[runconfigs[r['runconfig_id']]][machine] += r['total']
1065 self.participating_machines[runconfigs[r['runconfig_id']]][machine].add(r['machine_id'])
1066 machines_tmp.add(machine)
1068 # Order the machines and runconfigs
1069 runconfigs_ordered = sorted(runconfigs_tmp.keys(), key=lambda r: r.added_on, reverse=True)
1070 machines_ordered = sorted(machines_tmp, key=lambda m: str(m))
1072 # Create the final result structures
1073 self.runconfigs = OrderedDict() # [runconfig][machine] = Rate()
1074 self.machines = OrderedDict() # [machine][runconfig] = Rate()
1075 for machine in machines_ordered:
1076 self.machines[machine] = OrderedDict()
1077 for runconfig in runconfigs_ordered:
1078 # Add the runtime to both the runconfig-major and machine-major table
1079 self.runconfigs[runconfig] = OrderedDict()
1080 for machine in machines_ordered:
1081 total_time = runconfigs_tmp[runconfig][machine]
1082 machine_count = len(self.participating_machines[runconfig][machine])
1083 if average_per_machine:
1084 if machine_count > 0:
1085 actual_time = total_time / machine_count
1086 else:
1087 actual_time = timedelta()
1088 else:
1089 actual_time = total_time
1090 self.runconfigs[runconfig][machine] = self.machines[machine][runconfig] = actual_time
1092 def _machine_to_name(self, machine):
1093 counts = list()
1094 for runconfig in self.runconfigs:
1095 counts.append(len(self.participating_machines[runconfig][machine]))
1097 c_min = min(counts)
1098 c_max = max(counts)
1100 if c_max == 1:
1101 return str(machine)
1102 elif c_min == c_max:
1103 return "{} ({})".format(str(machine), c_min)
1104 else:
1105 return "{} ({}-{})".format(str(machine), c_min, c_max)
1107 @cached_property
1108 def chart(self):
1109 # The runconfigs are ordered from newest to oldest, reverse that
1110 runconfigs = list(reversed(self.runconfigs.keys()))
1112 chart_data = defaultdict(list)
1113 for runconfig in runconfigs:
1114 for machine in self.runconfigs[runconfig]:
1115 time = self.runconfigs[runconfig][machine]
1116 if time != timedelta():
1117 minutes = format(time.total_seconds() / 60, '.2f')
1118 else:
1119 minutes = "null"
1120 chart_data[self._machine_to_name(machine)].append(minutes)
1122 machines_colors = dict()
1123 for machine in self.machines:
1124 machines_colors[self._machine_to_name(machine)] = machine.color
1126 return LineChartData(chart_data, [r.name for r in runconfigs], machines_colors)
1128 @cached_property
1129 def longest_tests(self):
1130 TestRunTime = namedtuple("TestRunTime", ('test', 'machine', 'min', 'avg', 'max'))
1131 longest_tests = []
1133 db_results = TestResult.objects.filter(ts_run_id__in=self._tsr_ids)
1134 db_results = db_results.values("test_id", "ts_run__machine_id")
1135 db_results = db_results.annotate(d_avg=Avg("duration"),
1136 d_min=Min("duration"),
1137 d_max=Max("duration")).order_by("-d_max")
1138 db_results = db_results[:1000]
1140 tests = self._queryset_to_dict(Test, [r['test_id'] for r in db_results], "testsuite")
1141 machines = self._queryset_to_dict(Machine, [r['ts_run__machine_id'] for r in db_results], "aliases")
1143 # Create all the TestRunTime objects
1144 aliased_machines = defaultdict(list)
1145 for r in db_results:
1146 test = tests[r['test_id']]
1147 machine = machines[r['ts_run__machine_id']]
1148 runtime = TestRunTime(test, machine, r['d_min'], r['d_avg'], r['d_max'])
1150 if machine.aliases is None:
1151 longest_tests.append(runtime)
1152 else:
1153 aliased_machines[(test, machine.aliases)].append(runtime)
1155 # Deduplicate all the data for aliased machines
1156 for key, results in aliased_machines.items():
1157 test, machine = key
1159 # Aggregate all the values (Avg won't be super accurate, but good-enough)
1160 d_min = results[0].min
1161 avg_sum = timedelta()
1162 d_max = results[0].max
1163 for result in results:
1164 d_min = min(d_min, result.min)
1165 d_max = max(d_max, result.max)
1166 avg_sum += result.avg
1168 longest_tests.append(TestRunTime(results[0].test, machine, d_min, avg_sum / len(results), d_max))
1170 return sorted(longest_tests, key=lambda r: r.max, reverse=True)[:100]