Coverage for CIResults/serializers.py: 91%
390 statements
« prev ^ index » next coverage.py v7.6.10, created at 2025-01-20 14:33 +0000
« prev ^ index » next coverage.py v7.6.10, created at 2025-01-20 14:33 +0000
1import datetime
2from django.db import transaction
3from collections import namedtuple, OrderedDict
4from rest_framework import serializers
6from .run_import import JsonResult, RunConfigResults, TestsuiteResults, TestsuiteTestResult
7from .models import Bug, Component, Build, Test, Machine, RunConfigTag, RunConfig, ReplicationScript, UnknownFailure
8from .models import TestSuite, TextStatus, IssueFilter, MachineTag, BugTrackerAccount
9from .models import Issue, BugTracker
11from shortener.models import Shortener
14class UpdateMixin(serializers.ModelSerializer):
15 def get_extra_kwargs(self):
16 kwargs = super().get_extra_kwargs()
17 no_update_fields = getattr(self.Meta, "no_update_fields", None)
19 if self.instance and no_update_fields:
20 for field in no_update_fields:
21 kwargs.setdefault(field, {})
22 kwargs[field]["read_only"] = True
24 return kwargs
27class DynamicFieldsModelSerializer(serializers.ModelSerializer):
28 """
29 A ModelSerializer that takes an additional `extra_fields` argument that
30 controls which fields should be displayed. Subclass should define a
31 default_fields list.
32 """
33 default_fields = []
35 def __init__(self, *args, **kwargs):
36 extra_fields_to_serialize = kwargs.pop('extra_fields', None)
37 super().__init__(*args, **kwargs)
39 if extra_fields_to_serialize is not None:
40 # Drop extra fields that are not specified in the `extra_fields`
41 # argument.
42 default = set(self.default_fields)
43 existing = set(self.fields)
44 for field_name in existing - default - set(extra_fields_to_serialize):
45 self.fields.pop(field_name)
47 @classmethod
48 def extra_fields(cls) -> list[str]:
49 return list(set(cls.Meta.fields) - set(cls.default_fields))
52class RunConfigTagSerializer(serializers.ModelSerializer):
53 class Meta:
54 model = RunConfigTag
55 fields = ('id', 'description', 'url', 'public', '__str__')
56 read_only_fields = ('public', )
59class TestSuiteSerializer(serializers.ModelSerializer):
60 class Meta:
61 model = TestSuite
62 fields = ('id', '__str__')
65class TestSerializer(serializers.ModelSerializer):
66 testsuite = TestSuiteSerializer()
68 class Meta:
69 model = Test
70 fields = ('id', 'name', 'testsuite', 'public', 'vetted_on', 'first_runconfig', '__str__')
71 read_only_fields = ('public', )
74class MachineTagSerializer(serializers.ModelSerializer):
75 class Meta:
76 model = MachineTag
77 fields = ('id', 'name', 'public')
78 read_only_fields = ('added_on', )
81class MachineSerializer(serializers.ModelSerializer):
82 class Meta:
83 model = Machine
84 fields = ('id', 'public', 'vetted_on', '__str__')
85 read_only_fields = ('public', )
88class RestViewMachineSerializer(serializers.ModelSerializer):
89 tags = serializers.SlugRelatedField(many=True, slug_field='name', queryset=MachineTag.objects.all())
91 class Meta:
92 model = Machine
93 fields = ('id', 'name', 'description', 'public', 'vetted_on', 'aliases', 'tags')
96class ImportMachineError(Exception):
97 pass
100class ImportMachineSerializer(serializers.Serializer):
101 name = serializers.CharField(required=True)
102 description = serializers.CharField(required=False, allow_null=True)
103 public = serializers.BooleanField(required=False, default=False)
104 vetted = serializers.BooleanField(required=False, default=False)
105 alias = serializers.CharField(required=False, allow_null=True)
106 tags = serializers.ListSerializer(child=serializers.CharField(), required=False, default=[])
108 @transaction.atomic
109 def create(self, validated_data):
110 name = validated_data["name"]
111 description = validated_data.get("description")
112 alias = validated_data.get("alias")
113 if alias:
114 try:
115 alias_machine = Machine.objects.get(name=alias)
116 except Machine.DoesNotExist:
117 raise ImportMachineError(
118 "The machine this machine is supposed to alias does not exist. Create it first..."
119 )
120 else:
121 alias_machine = None
123 # List tags to show options
124 tags = {}
125 for tag in MachineTag.objects.all():
126 tags[tag.name] = tag
128 # Find the list of tags that do not exist yet, and create them
129 for tag_name in set(validated_data["tags"]) - set(tags.keys()):
130 tags[tag_name] = MachineTag.objects.create(name=tag_name, public=True)
132 # Now get/create the machine
133 try:
134 machine = Machine.objects.get(name=name)
135 except Machine.DoesNotExist:
136 machine = Machine.objects.create(name=name, public=validated_data["public"])
137 if validated_data["vetted"]:
138 machine.vet()
140 # Description
141 if description:
142 machine.description = description
144 # Alias
145 if alias_machine is not None:
146 machine.aliases = alias_machine
148 # Machine tags
149 tags_to_add = set(validated_data["tags"]) - set([mt.name for mt in machine.tags.all()])
150 for tag_name in tags_to_add:
151 machine.tags.add(tags[tag_name])
153 machine.save()
154 return machine
157class TextStatusSerializer(serializers.ModelSerializer):
158 testsuite = TestSuiteSerializer()
160 class Meta:
161 model = TextStatus
162 fields = ('id', 'name', 'testsuite', '__str__')
165class IssueFilterSerializer(serializers.ModelSerializer):
166 tags = RunConfigTagSerializer(many=True)
167 tests = TestSerializer(many=True)
168 machine_tags = MachineTagSerializer(many=True)
169 machines = MachineSerializer(many=True)
170 statuses = TextStatusSerializer(many=True)
172 class Meta:
173 model = IssueFilter
174 fields = ('id', 'description', 'tags', 'machines', 'machine_tags', 'tests', 'statuses',
175 'stdout_regex', 'stderr_regex', 'dmesg_regex', 'user_query', '__str__')
176 read_only_fields = ('added_on', )
179class RunConfigSerializer(serializers.ModelSerializer):
180 tags = serializers.SlugRelatedField(many=True, slug_field='name', queryset=RunConfigTag.objects.all())
181 builds = serializers.SlugRelatedField(many=True, slug_field='name', queryset=Build.objects.all())
183 class Meta:
184 model = RunConfig
185 fields = ('id', 'name', 'tags', 'url', 'added_on', 'builds', 'environment', 'temporary', '__str__')
186 read_only_fields = ('added_on', )
188 def validate(self, data):
189 components = {}
190 for build in data["builds"]:
191 if build.component not in components:
192 components[build.component] = build
193 else:
194 raise serializers.ValidationError(
195 f"Two builds ({components[build.component]} and {build}) cannot be from the same component"
196 )
197 return data
200class ImportTestResult(serializers.Serializer):
201 status = serializers.CharField()
202 duration = serializers.IntegerField(required=False, allow_null=True)
203 command = serializers.CharField(required=False, default="")
204 stdout = serializers.CharField(required=False, allow_null=True)
205 stderr = serializers.CharField(required=False, allow_null=True)
206 dmesg = serializers.CharField(required=False, allow_null=True)
207 url = serializers.URLField(required=False, allow_null=True)
208 start = serializers.DateTimeField(required=False, default=datetime.datetime.now())
211class ImportTestSuiteRunSerializer(serializers.Serializer):
212 runconfig_name = serializers.CharField(required=True)
213 test_results = serializers.DictField(
214 child=serializers.DictField(
215 child=serializers.DictField(child=ImportTestResult(), required=True), required=True), required=True
216 )
217 test_suite = serializers.CharField(required=True)
219 @transaction.atomic
220 def create(self, validated_data):
221 run_results = []
222 try:
223 test_suite = Build.objects.get(name=validated_data["test_suite"])
224 test_suite_results = TestsuiteResults(
225 runconfig=None,
226 name=test_suite.component.name,
227 build=test_suite,
228 format="json",
229 version=None,
230 result_url_pattern=validated_data.get("result_url_pattern")
231 )
232 except Build.DoesNotExist:
233 raise ValueError(f"Testsuite build {validated_data['test_suite']} does not exist")
234 for machine_name, raw_machine in validated_data.get("test_results").items():
235 for run_id, raw_run in raw_machine.items():
236 testsuite_test_results = []
237 for test_name, raw_test in raw_run.items():
238 testsuite_test_results.append(TestsuiteTestResult(
239 name=test_name,
240 status=raw_test["status"],
241 start_time=raw_test["start"],
242 duration=datetime.timedelta(seconds=raw_test.get("duration", 0)),
243 command=raw_test.get("command"),
244 stdout=raw_test.get("stdout"),
245 stderr=raw_test.get("stderr"),
246 dmesg=raw_test.get("dmesg"),
247 url=raw_test.get("url")
248 ))
249 run_results.append(JsonResult(test_suite_results, machine_name, int(run_id), testsuite_test_results))
251 rc = RunConfigResults(name=validated_data["runconfig_name"], parsed_results=run_results)
252 rc.commit_to_db()
253 return rc
256class MinimalMachineSerializer(serializers.ModelSerializer):
257 tags = serializers.StringRelatedField(many=True)
259 class Meta:
260 model = Machine
261 fields = ['name', 'tags']
264class MinimalRunConfigSerializer(serializers.ModelSerializer):
265 tags = serializers.StringRelatedField(many=True)
267 class Meta:
268 model = RunConfig
269 fields = ['name', 'tags']
272class UnknownFailureSerializer(DynamicFieldsModelSerializer):
273 test = serializers.StringRelatedField(source="result.test.name")
274 status = serializers.StringRelatedField(source="result.status.name")
275 dmesg = serializers.StringRelatedField(source="result.dmesg")
276 stdout = serializers.StringRelatedField(source="result.stdout")
277 stderr = serializers.StringRelatedField(source="result.stderr")
278 runconfig = MinimalRunConfigSerializer(source="result.ts_run.runconfig")
279 machine = MinimalMachineSerializer(source="result.ts_run.machine")
280 testsuite = serializers.StringRelatedField(source="result.ts_run.testsuite.name")
281 default_fields = ["id", "test", "status", "runconfig", "machine", "testsuite"]
283 class Meta:
284 model = UnknownFailure
285 fields = ["id", "test", "status", "dmesg", "stdout", "stderr", "runconfig", "machine", "testsuite"]
288class ComponentSerializer(serializers.ModelSerializer):
289 class Meta:
290 model = Component
291 fields = ('id', 'name', 'description', 'url', 'public', '__str__')
294class BuildSerializer(UpdateMixin, serializers.ModelSerializer):
295 component = serializers.SlugRelatedField(slug_field='name', queryset=Component.objects.all())
296 parents = serializers.SlugRelatedField(many=True, slug_field='name', queryset=Build.objects.all())
298 class Meta:
299 model = Build
300 fields = ('id', 'name', 'component', 'version', 'added_on', 'parents',
301 'repo_type', 'branch', 'repo', 'upstream_url', 'parameters',
302 'build_log', '__str__')
303 read_only_fields = ('id', 'added_on')
304 no_update_fields = ['name', 'component']
307class BuildMinimalSerializer(serializers.ModelSerializer):
308 class Meta:
309 model = Build
310 fields = ('id', 'name', 'added_on', 'parents', 'upstream_url', '__str__')
311 read_only_fields = ('id', 'added_on')
314class RunConfigResultsSerializer(serializers.Serializer):
315 __str__ = serializers.CharField(max_length=255, read_only=True)
316 is_failure = serializers.BooleanField(read_only=True)
317 all_failures_covered = serializers.BooleanField(read_only=True)
318 bugs_covering = serializers.SerializerMethodField()
320 def get_bugs_covering(self, obj):
321 ser = serializers.ListField(child=serializers.CharField(max_length=255, read_only=True))
322 return ser.to_representation([b.short_name for b in obj.bugs_covering])
325class RunConfigResultsDiffSerializer(serializers.Serializer):
326 testsuite = serializers.SerializerMethodField()
327 test = serializers.SerializerMethodField()
328 machine = serializers.SerializerMethodField()
329 result_from = RunConfigResultsSerializer(read_only=True)
330 result_to = RunConfigResultsSerializer(read_only=True)
332 def get_testsuite(self, obj):
333 ser = serializers.CharField(max_length=255, read_only=True)
334 return ser.to_representation(obj.testsuite.name)
336 def get_test(self, obj):
337 ser = serializers.CharField(max_length=255, read_only=True)
338 return ser.to_representation(obj.test.name)
340 def get_machine(self, obj):
341 ser = serializers.CharField(max_length=255, read_only=True)
342 return ser.to_representation(obj.machine.name)
345class BugTrackerSerializer(serializers.ModelSerializer):
346 class Meta:
347 model = BugTracker
348 fields = ('id', 'name', 'short_name', 'project', 'separator', 'url', 'tracker_type', 'polled',
349 'components_followed', 'components_followed_since', 'first_response_SLA')
350 read_only_fields = ('id', 'name', 'short_name', 'project', 'separator', 'url', 'tracker_type', 'polled',
351 'components_followed', 'components_followed_since', 'first_response_SLA')
354def serialize_bug(bug, new_comments=None):
355 def _date_formatter(date_field):
356 return str(date_field) if date_field is not None else None
358 runconfig_list = list(RunConfig.objects
359 .filter(testsuiterun__testresult__known_failure__matched_ifa__issue__bugs=bug)
360 .values()
361 .order_by('added_on'))
362 resp = {
363 'url': bug.url,
364 'bug_id': bug.bug_id,
365 'title': bug.title,
366 'description': bug.description,
367 'tracker': str(bug.tracker),
368 'created': _date_formatter(bug.created),
369 'updated': _date_formatter(bug.updated),
370 'polled': _date_formatter(bug.polled),
371 'closed': _date_formatter(bug.closed),
372 'creator': str(bug.creator),
373 'assignee': str(bug.assignee),
374 'product': bug.product,
375 'component': bug.component,
376 'priority': bug.priority,
377 'severity': bug.severity,
378 'features': bug.features_list,
379 'platforms': bug.platforms_list,
380 'status': bug.status,
381 'tags': bug.tags_list,
382 'runconfigs': runconfig_list,
383 'custom_fields': bug.custom_fields,
384 'new_comments': []
385 }
386 if new_comments:
387 for comm in new_comments:
388 person = comm.db_object.account.person
389 author = person.full_name if person.full_name else person.email
390 resp['new_comments'].append({'author': author,
391 'created': str(comm.db_object.created_on),
392 'body': comm.body})
393 return resp
396class BugCompleteSerializer(serializers.ModelSerializer):
397 tracker = BugTrackerSerializer()
399 class Meta:
400 model = Bug
401 fields = ('id', 'tracker', 'bug_id', 'parent', 'children', 'title', 'description', 'created',
402 'updated', 'polled', 'closed', 'creator', 'assignee', 'product', 'component',
403 'priority', 'severity', 'features', 'platforms', 'status', 'tags', 'comments_polled',
404 'flagged_as_update_pending_on', 'custom_fields')
405 read_only_fields = ('id', 'tracker', 'bug_id', 'parent', 'children', 'title', 'description', 'created',
406 'updated', 'polled', 'closed', 'creator', 'assignee', 'product', 'component',
407 'priority', 'severity', 'features', 'platforms', 'status', 'tags', 'comments_polled',
408 'flagged_as_update_pending_on', 'custom_fields')
411class BugMinimalSerializer(serializers.ModelSerializer):
412 class Meta:
413 model = Bug
414 fields = ('id', 'short_name', 'url')
415 read_only_fields = ('id', 'short_name', 'url')
418class ExecutionTimeSerializer(serializers.Serializer):
419 minimum = serializers.DurationField(read_only=True)
420 maximum = serializers.DurationField(read_only=True)
421 count = serializers.IntegerField(read_only=True)
424class RunConfigDiffSerializer(serializers.Serializer):
425 runcfg_from = RunConfigSerializer(read_only=True)
426 runcfg_to = RunConfigSerializer(read_only=True)
427 results = RunConfigResultsDiffSerializer(read_only=True, many=True)
428 new_tests = serializers.SerializerMethodField()
429 builds = serializers.SerializerMethodField()
430 bugs = BugMinimalSerializer(many=True)
431 status = serializers.CharField(max_length=10, read_only=True)
432 text = serializers.CharField(max_length=16000, read_only=True)
434 class BuildDiff2Serializer(serializers.Serializer):
435 component = ComponentSerializer(read_only=True)
436 from_build = BuildMinimalSerializer(read_only=True)
437 to_build = BuildMinimalSerializer(read_only=True)
439 class RunConfigDiffNewTestsSerializer(serializers.Serializer):
440 test = TestSerializer()
441 statuses = serializers.DictField(child=serializers.IntegerField(read_only=True))
442 exec_time = ExecutionTimeSerializer(read_only=True)
444 def get_builds(self, obj):
445 bd2 = namedtuple('BuildDiff2', ['component', 'from_build', 'to_build'])
446 build_diffs = {k: bd2(k, v.from_build, v.to_build) for k, v in obj.builds.items()}
447 dict_ser = serializers.DictField(child=self.BuildDiff2Serializer())
448 return dict_ser.to_representation(build_diffs)
450 def __statuses(self, statuses):
451 return {k.name: v for k, v in statuses.items()}
453 def get_new_tests(self, obj):
454 NT = namedtuple('NewTest', ['test', 'statuses', 'exec_time'])
455 new_tests = {k: NT(k, {k.name: v for k, v in v.to_statuses.items()}, v.to_exec_times)
456 for k, v in obj.new_tests.tests.items()}
457 dict_ser = serializers.DictField(child=self.RunConfigDiffNewTestsSerializer())
458 return dict_ser.to_representation(new_tests)
461class ReplicationScriptSerializer(serializers.ModelSerializer):
462 created_by = serializers.StringRelatedField()
463 source_tracker = serializers.StringRelatedField()
464 destination_tracker = serializers.StringRelatedField()
466 class Meta:
467 model = ReplicationScript
468 fields = ('name', 'created_by', 'created_on', 'source_tracker', 'destination_tracker', 'script')
471class KnownIssuesSerializer(serializers.Serializer):
472 id = serializers.IntegerField(read_only=True)
473 testsuite = serializers.SerializerMethodField()
474 machine = serializers.SerializerMethodField()
475 run_id = serializers.SerializerMethodField()
476 test = serializers.SerializerMethodField()
477 status = serializers.SerializerMethodField()
478 url = serializers.SerializerMethodField()
480 bugs = serializers.SerializerMethodField()
482 def __init__(self, *args, **kwargs):
483 super().__init__(*args, **kwargs)
485 # Cache the serializers for performance reasons
486 self._char_ser = serializers.CharField(max_length=255, read_only=True)
487 self._int_ser = serializers.IntegerField(read_only=True)
488 self._bug_min_ser = BugMinimalSerializer(many=True)
490 def get_run_id(self, obj):
491 return self._int_ser.to_representation(obj.result.ts_run.run_id)
493 def get_testsuite(self, obj):
494 return self._char_ser.to_representation(obj.result.test.testsuite.name)
496 def get_test(self, obj):
497 return self._char_ser.to_representation(obj.result.test.name)
499 def get_machine(self, obj):
500 return self._char_ser.to_representation(obj.result.ts_run.machine.name)
502 def get_status(self, obj):
503 return self._char_ser.to_representation(obj.result.status.name)
505 def get_url(self, obj):
506 return self._char_ser.to_representation(obj.result.url)
508 def get_bugs(self, obj):
509 return self._bug_min_ser.to_representation(obj.matched_ifa.issue.bugs.all())
512class BugTrackerAccountSerializer(serializers.ModelSerializer):
513 class Meta:
514 model = BugTrackerAccount
515 fields = ('id', 'is_developer')
516 read_only_fields = ('id', )
519class ShortenerSerializer(serializers.ModelSerializer):
520 class Meta:
521 model = Shortener
522 fields = ('id', 'shorthand', 'full', 'added_on', 'last_accessed')
523 read_only_fields = ('id', 'shorthand', 'full', 'added_on', 'last_accessed')
526class RateSerializer(serializers.Serializer):
527 count = serializers.IntegerField()
528 total = serializers.IntegerField()
529 percent = serializers.FloatField()
532class BugSerializer(serializers.ModelSerializer):
533 class Meta:
534 model = Bug
535 fields = ('short_name', 'title', 'url')
536 read_only_fields = ('short_name', 'title', 'url')
539class RestIssueSerializer(serializers.ModelSerializer):
540 class Meta:
541 model = Issue
542 fields = ('__all__')
543 read_only_fields = ('id', 'added_on', 'archived_on', 'runconfigs_covered_count', 'runconfigs_affected_count',
544 'last_seen', 'added_by', 'archived_by', 'last_seen_runconfig')
546 bugs = BugSerializer(many=True)
549class IssueSerializer(serializers.ModelSerializer):
550 class Meta:
551 model = Issue
552 fields = ('id', 'bugs', 'expected')
553 read_only_fields = ('id', 'bugs', 'expected')
555 bugs = BugSerializer(many=True)
558class IssueSerializerMinimal(serializers.ModelSerializer):
559 class Meta:
560 model = Issue
561 fields = ('id', )
562 read_only_fields = ('id', )
565def serialize_issue_hitrate(issues, minimal=False):
566 Serializer = IssueSerializerMinimal if minimal else IssueSerializer
568 ret = []
569 for issue, rate in issues.items():
570 val = Serializer(issue).data
571 val['hit_rate'] = RateSerializer(rate).data
572 ret.append(val)
573 return ret
576def serialize_MetricPassRatePerRunconfig(history):
577 runconfigs = OrderedDict()
578 for runconfig, _statuses in history.runconfigs.items():
579 runconfigs[str(runconfig)] = OrderedDict()
580 for status, rate in _statuses.items():
581 runconfigs[str(runconfig)][str(status)] = RateSerializer(rate).data
583 statuses = OrderedDict()
584 for status, _runconfigs in history.statuses.items():
585 statuses[str(status)] = OrderedDict()
586 for runconfig, rate in _runconfigs.items():
587 statuses[str(status)][str(runconfig)] = RateSerializer(rate).data
589 return {
590 "runconfigs": runconfigs,
591 "statuses": statuses,
592 "most_hit_issues": serialize_issue_hitrate(history.most_hit_issues),
593 "query_key": history.query.query_key,
594 }
597class PassRateStatisticsSerializer(serializers.Serializer):
598 passrate = RateSerializer()
599 runrate = RateSerializer()
600 discarded_rate = RateSerializer()
601 notrun_rate = RateSerializer()
604def serialize_MetricPassRatePerTest(metric_passrate):
605 discarded_status = "discarded (expected)"
607 tests = OrderedDict()
608 for test, results in metric_passrate.tests.items():
609 if results.is_fully_discarded:
610 tests[str(test)] = {
611 "status": discarded_status,
612 "is_pass": False,
613 "is_run": False,
614 "duration": str(results.duration),
615 "issues_hit": serialize_issue_hitrate(results.issue_occurence_rates, minimal=True),
616 }
617 else:
618 tests[str(test)] = {
619 "status": str(results.overall_result),
620 "is_pass": results.is_pass,
621 "is_run": not results.overall_result.is_notrun,
622 "duration": str(results.duration),
623 "issues_hit": serialize_issue_hitrate(results.issue_occurence_rates, minimal=True),
624 }
626 statuses = OrderedDict()
627 for status, rate in metric_passrate.statuses.items():
628 statuses[str(status)] = RateSerializer(rate).data
630 return {
631 "tests": tests,
632 "statuses": statuses,
633 "discarded_status": discarded_status,
634 "machines": [str(m) for m in metric_passrate.machines],
635 "runconfigs": RunConfigSerializer(metric_passrate.runconfigs, many=True).data,
636 "raw_statistics": PassRateStatisticsSerializer(metric_passrate.raw_statistics).data,
637 "statistics": PassRateStatisticsSerializer(metric_passrate.statistics).data,
638 "most_hit_issues": serialize_issue_hitrate(metric_passrate.most_hit_issues),
639 "uncovered_failure_rate": RateSerializer(metric_passrate.uncovered_failure_rate).data,
640 "notrun_rate": RateSerializer(metric_passrate.notrun_rate).data,
641 "most_interrupting_issues": serialize_issue_hitrate(metric_passrate.most_interrupting_issues),
642 "unknown_failure_interruption_rate": RateSerializer(metric_passrate.unknown_failure_interruption_rate).data,
643 "unexplained_interruption_rate": RateSerializer(metric_passrate.unexplained_interruption_rate).data,
644 "query_key": metric_passrate.query.query_key,
645 }