Coverage for CIResults / rest_views.py: 100%
316 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-03-04 08:33 +0000
1from django.shortcuts import get_object_or_404
2from django.http import HttpResponse, JsonResponse
3from django.db import transaction, models
4import django_filters
6from rest_framework.decorators import action, api_view, permission_classes
7from rest_framework.generics import RetrieveAPIView, ListCreateAPIView
8from rest_framework.response import Response
9from rest_framework.pagination import PageNumberPagination
10from rest_framework import status, viewsets, permissions, mixins, serializers
11from django_filters import rest_framework as filters
12from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiParameter
13from drf_spectacular.types import OpenApiTypes
15from .serializers import (
16 IssueAddFilterSerializer,
17 ImportTestSuiteRunSerializer,
18 IssueFilterSerializer,
19 RunConfigSerializer,
20 TextStatusSerializer,
21)
22from .serializers import BuildSerializer, TestSerializer, BugTrackerAccountSerializer, KnownIssuesSerializer
23from .serializers import serialize_MetricPassRatePerRunconfig, serialize_MetricPassRatePerTest, BugTrackerSerializer
24from .serializers import BugCompleteSerializer, ShortenerSerializer, ImportMachineSerializer, ComponentSerializer
25from .serializers import RunConfigDiffSerializer, RestIssueSerializer
26from .serializers import UnknownFailureSerializer
27from .filtering import QueryCreator
29from .models import Component, Build, Test, Machine, RunConfigTag, RunConfig, TextStatus, Bug, TestResult
30from .models import IssueFilter, Issue, KnownFailure, MachineTag, BugTrackerAccount, BugTracker, UnknownFailure
31from .metrics import MetricPassRatePerRunconfig, MetricPassRatePerTest
33from shortener.models import Shortener
35import json
36import re
39def get_obj_by_id_or_name(model, key):
40 # FIXME: There are some corner cases, where this function is faulty, e.g. when the name of object contains only
41 # numbers, and the number is bigger than the model's maximal ID this function raises model.DoesNotExist
42 # exception - client receives response with status code 500.
43 # Fixing it with catching the exception will only create another issue, where users could get objects,
44 # which they didn't ask for, e.g. they provide the name that contains only numbers, the object
45 # doesn't exist, although they still get some "random" object with id equal to the name.
46 try:
47 return model.objects.get(pk=int(key))
48 except ValueError:
49 pass
51 return get_object_or_404(model, name=key)
54def object_vet(model, pk):
55 obj = get_object_or_404(model, pk=pk)
56 if not obj.vetted:
57 obj.vet()
58 return Response(status=status.HTTP_200_OK)
61def object_suppress(model, pk):
62 obj = get_object_or_404(model, pk=pk)
63 if obj.vetted:
64 obj.suppress()
65 return Response(status=status.HTTP_200_OK)
68class CustomPagination(PageNumberPagination):
69 page_size = 100
70 page_size_query_param = 'page_size'
71 max_page_size = None
73 def get_page_size(self, request):
74 if self.page_size_query_param:
75 try:
76 page_size = int(request.query_params.get(self.page_size_query_param, self.page_size))
77 except ValueError:
78 return self.page_size
80 # Clamp the maximum size
81 if self.max_page_size:
82 page_size = min(page_size, self.max_page_size)
84 # Do not set any limits if the page size is null or negative
85 if page_size <= 0:
86 return None
87 else:
88 return page_size
90 return self.page_size
93class IssueViewSet(viewsets.ReadOnlyModelViewSet):
94 queryset = Issue.objects.all().order_by('-id')
95 serializer_class = RestIssueSerializer
97 def patch(self, request, pk):
98 issue = Issue.objects.get(pk=pk)
99 serializer = RestIssueSerializer(issue, data=request.data, partial=True)
100 if serializer.is_valid():
101 serializer.save()
102 return Response(data=serializer.data, status=status.HTTP_200_OK)
103 return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST)
105 @classmethod
106 def _execute_action(cls, request, pk, permsission_name, action):
107 if not request.user.has_perm(permsission_name):
108 return Response(data={"message": f"User {request.user} doesn't have sufficient permissions"},
109 status=status.HTTP_401_UNAUTHORIZED)
110 try:
111 issue = get_object_or_404(Issue, pk=pk)
112 action(issue)
113 except Exception as err:
114 return Response(data={"message": str(err)}, status=status.HTTP_400_BAD_REQUEST)
115 return Response(status=status.HTTP_200_OK)
117 @action(detail=True)
118 def archive(self, request, pk=None):
119 return self._execute_action(request, pk, "CIResults.archive_issue", lambda issue: issue.archive(request.user))
121 @action(detail=True)
122 def restore(self, request, pk=None):
123 return self._execute_action(request, pk, "CIResults.restore_issue", lambda issue: issue.restore())
125 @extend_schema(
126 description=("Add filters to an issue"),
127 parameters=[
128 OpenApiParameter(
129 name="id",
130 description="Id of the issue",
131 location=OpenApiParameter.PATH,
132 ),
133 ],
134 request=IssueAddFilterSerializer,
135 responses={
136 status.HTTP_200_OK: OpenApiTypes.NONE,
137 status.HTTP_400_BAD_REQUEST: OpenApiTypes.STR,
138 },
139 )
140 @action(detail=True, methods=["post"], url_path="add/filters", url_name="add-filters")
141 def add_filters(self, request, pk=None) -> Response:
142 filters_serializer: IssueAddFilterSerializer = IssueAddFilterSerializer(data=request.data)
143 filters_serializer.is_valid(raise_exception=True)
144 return self._execute_action(
145 request,
146 pk,
147 "CIResults.change_issue",
148 lambda issue: issue.add_filters(filters_serializer.validated_data["filters"], request.user)
149 )
152class IssueFilterViewSet(viewsets.ModelViewSet):
153 queryset = IssueFilter.objects.all().order_by('-id').prefetch_related('tags', 'tests__testsuite',
154 'tests__first_runconfig',
155 'machine_tags', 'machines',
156 'statuses__testsuite')
157 serializer_class = IssueFilterSerializer
158 pagination_class = CustomPagination
160 def __check_list__(self, request_data, field, field_name, db_class, errors):
161 objects = set(request_data.get(field, []))
163 objects_db = dict()
164 for obj in db_class.objects.filter(id__in=objects):
165 objects_db[obj.id] = obj
167 if len(objects) != len(objects_db):
168 errors.append("At least one {} does not exist".format(field_name))
170 return objects, objects_db
172 def __get_or_None__(self, klass, field, request_dict, errors):
173 obj = None
174 if field in request_dict:
175 obj_id = request_dict[field]
177 # Do not consider empty strings as meaning a valid value
178 if isinstance(obj_id, str) and len(obj_id) == 0:
179 return None
181 # Convert the id to an int or fail
182 try:
183 obj_id = int(obj_id)
184 except Exception:
185 errors.append("The field '{}' needs to be an integer".format(field))
186 return None
188 # Try getting the object
189 obj = klass.objects.filter(id=obj_id).first()
190 if obj is None:
191 errors.append("The object referenced by '{}' does not exist".format(field))
193 return obj
195 def get_queryset(self):
196 queryset = self.queryset
197 if description := self.request.query_params.get("description"):
198 queryset = queryset.filter(description__contains=description)
199 return queryset
201 @transaction.atomic
202 def create(self, request):
203 errors = []
204 if len(request.data.get('description', '')) == 0:
205 errors.append("The field 'description' cannot be empty")
207 # Check if the filter should replace another one
208 edit_filter = self.__get_or_None__(IssueFilter, 'edit_filter',
209 request.data, errors)
210 edit_issue = self.__get_or_None__(Issue, 'edit_issue',
211 request.data, errors)
213 # Check that all the tags, machines, tests, and statuses are present
214 tags, tags_db = self.__check_list__(request.data, "tags", "tag", RunConfigTag, errors)
215 machine_tags, machine_tags_db = self.__check_list__(request.data, "machine_tags", "machine tag",
216 MachineTag, errors)
217 machines, machines_db = self.__check_list__(request.data, "machines", "machine", Machine, errors)
218 tests, tests_db = self.__check_list__(request.data, "tests", "test", Test, errors)
219 statuses, statuses_db = self.__check_list__(request.data, "statuses", "status", TextStatus, errors)
221 # Check the regular expressions
222 for field in ['stdout_regex', 'stderr_regex', 'dmesg_regex']:
223 try:
224 re.compile(request.data.get(field, ""), re.DOTALL)
225 except Exception:
226 errors.append("The field '{}' does not contain a valid regular expression".format(field))
228 # Create the object or fail depending on whether we got errors or not
229 if len(errors) == 0:
230 filter = IssueFilter.objects.create(description=request.data.get('description'),
231 stdout_regex=request.data.get('stdout_regex', ""),
232 stderr_regex=request.data.get('stderr_regex', ""),
233 dmesg_regex=request.data.get('dmesg_regex', ""),
234 user_query=request.data.get('user_query', ""))
236 filter.tags.add(*tags_db)
237 filter.machines.add(*machines_db)
238 filter.machine_tags.add(*machine_tags_db)
239 filter.tests.add(*tests_db)
240 filter.statuses.add(*statuses_db)
242 # If this filter is supposed to replace another filter
243 if edit_filter is not None:
244 if edit_issue is not None:
245 edit_issue.replace_filter(edit_filter, filter, request.user)
246 else:
247 edit_filter.replace(filter, request.user)
249 serializer = IssueFilterSerializer(filter)
250 return Response(serializer.data, status=status.HTTP_201_CREATED)
251 else:
252 return Response(errors, status=status.HTTP_400_BAD_REQUEST)
255class RunConfigFilter(filters.FilterSet):
256 class Meta:
257 model = RunConfig
258 fields = {
259 "name": ["exact", "contains"],
260 "builds__name": ["exact", "contains"],
261 }
264@extend_schema_view(
265 retrieve=extend_schema(
266 parameters=[
267 OpenApiParameter(
268 name="id",
269 description="A unique ID or name identifying this runconfig.",
270 location=OpenApiParameter.PATH,
271 )
272 ],
273 responses={status.HTTP_200_OK: RunConfigSerializer}
274 ),
275)
276class RunConfigViewSet(mixins.CreateModelMixin,
277 mixins.ListModelMixin,
278 mixins.RetrieveModelMixin,
279 viewsets.GenericViewSet):
280 queryset = RunConfig.objects.all().order_by('-id')
281 serializer_class = RunConfigSerializer
282 lookup_value_regex = r"[\w.-]+"
283 filterset_class = RunConfigFilter
285 def get_object(self):
286 return get_obj_by_id_or_name(RunConfig, self.kwargs.get("pk"))
288 @classmethod
289 def known_failures_serialized(cls, runcfg):
290 f = KnownFailure.objects.filter(result__ts_run__runconfig=runcfg)
291 failures = f.prefetch_related('result__status', 'result__test', 'result__test__testsuite',
292 'result__ts_run__machine', 'matched_ifa__issue__bugs',
293 'matched_ifa__issue__bugs__tracker')
294 return KnownIssuesSerializer(failures, read_only=True, many=True)
296 @extend_schema(
297 parameters=[
298 OpenApiParameter(
299 name="id",
300 description="A unique ID or name identifying this runconfig.",
301 location=OpenApiParameter.PATH,
302 )
303 ],
304 responses={status.HTTP_200_OK: KnownIssuesSerializer}
305 )
306 @action(detail=True)
307 def known_failures(self, request, pk=None):
308 runcfg = self.get_object()
309 return Response(self.known_failures_serialized(runcfg).data, status=status.HTTP_200_OK)
311 @extend_schema(
312 description="Compare two runconfigs",
313 parameters=[
314 OpenApiParameter(
315 name="id",
316 description="Name or ID of of the RunConfig used for the test suite run",
317 location=OpenApiParameter.PATH
318 ),
319 OpenApiParameter(
320 name="to",
321 description="Id or name of RunConfig to compare",
322 required=True
323 ),
324 OpenApiParameter(
325 name="no_compress",
326 description="Should not compress comparison results",
327 type=bool
328 ),
329 OpenApiParameter(
330 name="summary",
331 description="Return raw text summary (to enable add this parameter to request)",
332 )
333 ],
334 responses={status.HTTP_200_OK: RunConfigDiffSerializer}
335 )
336 @action(detail=True)
337 def compare(self, request, pk=None):
338 runcfg_from = self.get_object()
339 runcfg_to = get_obj_by_id_or_name(RunConfig, request.GET.get('to'))
340 no_compress = serializers.BooleanField().to_representation(request.GET.get('no_compress'))
342 diff = runcfg_from.compare(runcfg_to, no_compress=no_compress)
343 if request.GET.get('summary') is None:
344 serializer = RunConfigDiffSerializer(diff)
345 return Response(serializer.data, status=status.HTTP_200_OK)
346 else:
347 return HttpResponse(diff.text)
349 @extend_schema(
350 description=(
351 "Create test suite run object"
352 ),
353 parameters=[
354 OpenApiParameter(
355 name="id",
356 description="Id or name of base RunConfig",
357 location=OpenApiParameter.PATH
358 ),
359 ],
360 request=ImportTestSuiteRunSerializer,
361 responses={
362 status.HTTP_200_OK: ImportTestSuiteRunSerializer,
363 status.HTTP_400_BAD_REQUEST: OpenApiTypes.STR,
364 },
365 )
366 @action(detail=True, methods=["post"], url_path='testsuiterun')
367 def import_test_suite_run(self, request, pk=None):
368 try:
369 request.data["runconfig_name"] = self.get_object().name
370 serializer = ImportTestSuiteRunSerializer(data=request.data)
371 if serializer.is_valid():
372 serializer.save()
373 return Response(serializer.data, status=status.HTTP_200_OK)
374 else:
375 return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
376 except Exception as error:
377 return Response(str(error), status=status.HTTP_400_BAD_REQUEST)
380class ComponentViewSet(viewsets.ReadOnlyModelViewSet):
381 queryset = Component.objects.all().order_by('-id')
382 serializer_class = ComponentSerializer
385class BuildViewSet(mixins.CreateModelMixin,
386 mixins.ListModelMixin,
387 mixins.RetrieveModelMixin,
388 mixins.UpdateModelMixin,
389 viewsets.GenericViewSet):
390 queryset = Build.objects.all().order_by('-id')
391 serializer_class = BuildSerializer
393 @extend_schema(
394 parameters=[
395 OpenApiParameter(
396 name="id",
397 description="A unique ID or name identifying this build.",
398 location=OpenApiParameter.PATH,
399 )
400 ]
401 )
402 def retrieve(self, request, pk=None):
403 build = get_obj_by_id_or_name(Build, pk)
404 serializer = BuildSerializer(build)
405 return Response(serializer.data)
408class MachineViewSet(
409 mixins.ListModelMixin,
410 mixins.CreateModelMixin,
411 mixins.RetrieveModelMixin,
412 viewsets.GenericViewSet,
413):
414 queryset = Machine.objects.all()
415 serializer_class = ImportMachineSerializer
416 filterset_fields = {
417 "id": ["exact"],
418 "name": ["exact"],
419 "public": ["exact"],
420 "vetted_on": ["isnull"],
421 }
423 @action(detail=True, methods=["post"])
424 def vet(self, request, pk):
425 return object_vet(Machine, pk)
427 @action(detail=True, methods=["post"])
428 def suppress(self, request, pk):
429 return object_suppress(Machine, pk)
432class TestFilter(filters.FilterSet):
433 class Meta:
434 model = Test
435 fields = {
436 'id': ['exact'],
437 'name': ['exact'],
438 'testsuite': ['exact'],
439 'public': ['exact'],
440 'added_on': ['lte', 'gte'],
441 'vetted_on': ['isnull'],
442 }
443 filter_overrides = {
444 models.DateTimeField: {
445 'filter_class': django_filters.IsoDateTimeFilter
446 },
447 }
450class TestSet(viewsets.ReadOnlyModelViewSet):
451 queryset = Test.objects.all().order_by('-id')
452 serializer_class = TestSerializer
453 filterset_class = TestFilter
455 @action(detail=True, methods=["post"])
456 def vet(self, request, pk):
457 return object_vet(Test, pk)
459 @action(detail=True, methods=["post"])
460 def suppress(self, request, pk):
461 return object_suppress(Test, pk)
464class UnknownFailureViewSet(viewsets.ReadOnlyModelViewSet):
465 queryset = UnknownFailure.objects.all().order_by('-id')
466 serializer_class = UnknownFailureSerializer
467 pagination_class = CustomPagination
469 EXTRA_FIELDS_ARG = OpenApiParameter(
470 name="extra_fields",
471 description=(
472 f"Comma-separated list of fields to expand (available choices: "
473 f"{', '.join(serializer_class.extra_fields())})"
474 ),
475 type=str,
476 )
478 @extend_schema(description="Retrieve a single unknown failure", parameters=[EXTRA_FIELDS_ARG])
479 def retrieve(self, request, pk):
480 extra_fields = request.query_params.get('extra_fields', '').split(',')
481 failure = get_object_or_404(UnknownFailure, pk=pk)
482 serializer = self.serializer_class(failure, extra_fields=extra_fields)
483 return Response(serializer.data)
485 @extend_schema(description="List all unknown failures", parameters=[EXTRA_FIELDS_ARG])
486 def list(self, request):
487 page = self.paginate_queryset(self.queryset)
488 extra_fields = request.query_params.get('extra_fields', '').split(',')
489 serializer = self.serializer_class(page, extra_fields=extra_fields, many=True)
490 return self.get_paginated_response(serializer.data)
493class TextStatusViewSet(viewsets.ReadOnlyModelViewSet):
494 queryset = TextStatus.objects.all().order_by('-id')
495 serializer_class = TextStatusSerializer
497 @action(detail=True)
498 def vet(self, request, pk):
499 return object_vet(TextStatus, pk)
501 @action(detail=True)
502 def suppress(self, request, pk):
503 return object_suppress(TextStatus, pk)
506class BugTrackerViewSet(viewsets.ReadOnlyModelViewSet):
507 queryset = BugTracker.objects.all().order_by('-id')
508 serializer_class = BugTrackerSerializer
511class BugViewSet(RetrieveAPIView):
512 serializer_class = BugCompleteSerializer
513 queryset = Bug.objects.all()
515 def _get_bugtracker(self):
516 tracker = self.kwargs.get('tracker')
518 try:
519 return BugTracker.objects.get(pk=int(tracker))
520 except (ValueError, BugTracker.DoesNotExist):
521 pass
523 try:
524 return BugTracker.objects.get(name=tracker)
525 except BugTracker.DoesNotExist:
526 pass
528 return get_object_or_404(BugTracker, short_name=tracker)
530 def retrieve(self, request, *args, **kwargs):
531 bug = get_object_or_404(Bug, tracker=self._get_bugtracker(), bug_id=kwargs.get('bug_id'))
532 serializer = self.get_serializer(bug)
533 return Response(serializer.data)
536class BugTrackerAccountViewSet(viewsets.ModelViewSet):
537 # WARNING: we do not yet perform access control for setting who is a user or developer because of the limited damage
538 # this can cause and the annoyance of having to ask large group of users to authenticate then be granted the
539 # privilege to change the roles
540 permission_classes = []
541 authentication_classes = []
543 queryset = BugTrackerAccount.objects.all().order_by('-id')
544 serializer_class = BugTrackerAccountSerializer
546 http_method_names = ['get', 'patch']
549class ShortenerViewSet(ListCreateAPIView, viewsets.GenericViewSet):
550 # WARNING: No access control is performed because these objects can be created simply by navigating the website
551 permission_classes = []
552 authentication_classes = []
554 queryset = Shortener.objects.all().order_by('-id')
555 serializer_class = ShortenerSerializer
557 def create(self, request, *args, **kwargs):
558 if request.method != "POST" or request.content_type != "application/json":
559 raise ValueError("Only JSON POST requests are supported")
561 data = json.loads(request.body)
562 fulls = data.get('full')
563 if fulls is None:
564 raise ValueError("Missing the field 'full' which should contain the full text to be shortened")
566 if isinstance(fulls, list):
567 shorts = [Shortener.get_or_create(full=f) for f in fulls]
568 serializer = self.get_serializer(shorts, many=True)
569 else:
570 short = Shortener.get_or_create(full=fulls)
571 serializer = self.get_serializer(short)
573 return JsonResponse(serializer.data, safe=False)
576@api_view()
577@permission_classes((permissions.AllowAny,))
578def metrics_passrate_per_runconfig_view(request):
579 user_query = QueryCreator(request, TestResult).request_to_query()
580 history = MetricPassRatePerRunconfig(user_query)
581 return Response(serialize_MetricPassRatePerRunconfig(history))
584@api_view(['GET', 'POST'])
585@permission_classes((permissions.AllowAny,))
586def metrics_passrate_per_test_view(request):
587 user_query = QueryCreator(request, TestResult).request_to_query()
588 passrate = MetricPassRatePerTest(user_query)
589 return Response(serialize_MetricPassRatePerTest(passrate))