Coverage for CIResults/rest_views.py: 100%
321 statements
« prev ^ index » next coverage.py v7.6.9, created at 2024-12-19 09:20 +0000
1from django.shortcuts import get_object_or_404
2from django.http import HttpResponse, JsonResponse
3from django.db import transaction, models
4import django_filters
6from rest_framework.decorators import action, api_view, permission_classes
7from rest_framework.generics import RetrieveAPIView, ListCreateAPIView
8from rest_framework.response import Response
9from rest_framework.pagination import PageNumberPagination
10from rest_framework import status, viewsets, permissions, mixins
11from django_filters import rest_framework as filters
12from drf_spectacular.utils import extend_schema, OpenApiParameter
14from .serializers import ImportTestSuiteRunSerializer, IssueFilterSerializer, RunConfigSerializer, TextStatusSerializer
15from .serializers import BuildSerializer, TestSerializer, BugTrackerAccountSerializer, KnownIssuesSerializer
16from .serializers import serialize_MetricPassRatePerRunconfig, serialize_MetricPassRatePerTest, BugTrackerSerializer
17from .serializers import BugCompleteSerializer, ShortenerSerializer, RestViewMachineSerializer, ComponentSerializer
18from .serializers import ImportMachineError, ImportMachineSerializer, RunConfigDiffSerializer, RestIssueSerializer
19from .serializers import UnknownFailureSerializer
20from .filtering import QueryCreator
22from .models import Component, Build, Test, Machine, RunConfigTag, RunConfig, TextStatus, Bug, TestResult
23from .models import IssueFilter, Issue, KnownFailure, MachineTag, BugTrackerAccount, BugTracker, UnknownFailure
24from .metrics import MetricPassRatePerRunconfig, MetricPassRatePerTest
26from shortener.models import Shortener
28import json
29import re
32def get_obj_by_id_or_name(model, key):
33 try:
34 return model.objects.get(pk=int(key))
35 except ValueError:
36 pass
38 return get_object_or_404(model, name=key)
41def object_vet(model, pk):
42 obj = get_object_or_404(model, pk=pk)
43 if not obj.vetted:
44 obj.vet()
45 return Response(status=status.HTTP_200_OK)
48def object_suppress(model, pk):
49 obj = get_object_or_404(model, pk=pk)
50 if obj.vetted:
51 obj.suppress()
52 return Response(status=status.HTTP_200_OK)
55class CustomPagination(PageNumberPagination):
56 page_size = 100
57 page_size_query_param = 'page_size'
58 max_page_size = None
60 def get_page_size(self, request):
61 if self.page_size_query_param:
62 try:
63 page_size = int(request.query_params.get(self.page_size_query_param, self.page_size))
64 except ValueError:
65 return self.page_size
67 # Clamp the maximum size
68 if self.max_page_size:
69 page_size = min(page_size, self.max_page_size)
71 # Do not set any limits if the page size is null or negative
72 if page_size <= 0:
73 return None
74 else:
75 return page_size
77 return self.page_size
80class IssueViewSet(viewsets.ReadOnlyModelViewSet):
81 queryset = Issue.objects.all().order_by('-id')
82 serializer_class = RestIssueSerializer
84 def patch(self, request, pk):
85 issue = Issue.objects.get(pk=pk)
86 serializer = RestIssueSerializer(issue, data=request.data, partial=True)
87 if serializer.is_valid():
88 serializer.save()
89 return Response(data=serializer.data, status=status.HTTP_200_OK)
90 return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST)
92 @classmethod
93 def _execute_action(cls, request, pk, permsission_name, action):
94 if not request.user.has_perm(permsission_name):
95 return Response(data={"message": f"User {request.user} doesn't have sufficient permissions"},
96 status=status.HTTP_401_UNAUTHORIZED)
97 try:
98 issue = get_object_or_404(Issue, pk=pk)
99 action(issue)
100 except Exception as err:
101 return Response(data={"message": str(err)}, status=status.HTTP_400_BAD_REQUEST)
102 return Response(status=status.HTTP_200_OK)
104 @action(detail=True)
105 def archive(self, request, pk=None):
106 return self._execute_action(request, pk, "CIResults.archive_issue", lambda issue: issue.archive(request.user))
108 @action(detail=True)
109 def restore(self, request, pk=None):
110 return self._execute_action(request, pk, "CIResults.restore_issue", lambda issue: issue.restore())
113class IssueFilterViewSet(viewsets.ModelViewSet):
114 queryset = IssueFilter.objects.all().order_by('-id').prefetch_related('tags', 'tests__testsuite',
115 'tests__first_runconfig',
116 'machine_tags', 'machines',
117 'statuses__testsuite')
118 serializer_class = IssueFilterSerializer
119 pagination_class = CustomPagination
121 def __check_list__(self, request_data, field, field_name, db_class, errors):
122 objects = set(request_data.get(field, []))
124 objects_db = dict()
125 for obj in db_class.objects.filter(id__in=objects):
126 objects_db[obj.id] = obj
128 if len(objects) != len(objects_db):
129 errors.append("At least one {} does not exist".format(field_name))
131 return objects, objects_db
133 def __get_or_None__(self, klass, field, request_dict, errors):
134 obj = None
135 if field in request_dict:
136 obj_id = request_dict[field]
138 # Do not consider empty strings as meaning a valid value
139 if isinstance(obj_id, str) and len(obj_id) == 0:
140 return None
142 # Convert the id to an int or fail
143 try:
144 obj_id = int(obj_id)
145 except Exception:
146 errors.append("The field '{}' needs to be an integer".format(field))
147 return None
149 # Try getting the object
150 obj = klass.objects.filter(id=obj_id).first()
151 if obj is None:
152 errors.append("The object referenced by '{}' does not exist".format(field))
154 return obj
156 def get_queryset(self):
157 queryset = self.queryset
158 if description := self.request.query_params.get("description"):
159 queryset = queryset.filter(description__contains=description)
160 return queryset
162 @transaction.atomic
163 def create(self, request):
164 errors = []
165 if len(request.data.get('description', '')) == 0:
166 errors.append("The field 'description' cannot be empty")
168 # Check if the filter should replace another one
169 edit_filter = self.__get_or_None__(IssueFilter, 'edit_filter',
170 request.data, errors)
171 edit_issue = self.__get_or_None__(Issue, 'edit_issue',
172 request.data, errors)
174 # Check that all the tags, machines, tests, and statuses are present
175 tags, tags_db = self.__check_list__(request.data, "tags", "tag", RunConfigTag, errors)
176 machine_tags, machine_tags_db = self.__check_list__(request.data, "machine_tags", "machine tag",
177 MachineTag, errors)
178 machines, machines_db = self.__check_list__(request.data, "machines", "machine", Machine, errors)
179 tests, tests_db = self.__check_list__(request.data, "tests", "test", Test, errors)
180 statuses, statuses_db = self.__check_list__(request.data, "statuses", "status", TextStatus, errors)
182 # Check the regular expressions
183 for field in ['stdout_regex', 'stderr_regex', 'dmesg_regex']:
184 try:
185 re.compile(request.data.get(field, ""), re.DOTALL)
186 except Exception:
187 errors.append("The field '{}' does not contain a valid regular expression".format(field))
189 # Create the object or fail depending on whether we got errors or not
190 if len(errors) == 0:
191 filter = IssueFilter.objects.create(description=request.data.get('description'),
192 stdout_regex=request.data.get('stdout_regex', ""),
193 stderr_regex=request.data.get('stderr_regex', ""),
194 dmesg_regex=request.data.get('dmesg_regex', ""),
195 user_query=request.data.get('user_query', ""))
197 filter.tags.add(*tags_db)
198 filter.machines.add(*machines_db)
199 filter.machine_tags.add(*machine_tags_db)
200 filter.tests.add(*tests_db)
201 filter.statuses.add(*statuses_db)
203 # If this filter is supposed to replace another filter
204 if edit_filter is not None:
205 if edit_issue is not None:
206 edit_issue.replace_filter(edit_filter, filter, request.user)
207 else:
208 edit_filter.replace(filter, request.user)
210 serializer = IssueFilterSerializer(filter)
211 return Response(serializer.data, status=status.HTTP_201_CREATED)
212 else:
213 return Response(errors, status=status.HTTP_400_BAD_REQUEST)
216class RunConfigFilter(filters.FilterSet):
217 class Meta:
218 model = RunConfig
219 fields = {
220 "name": ["exact", "contains"],
221 "builds__name": ["exact", "contains"],
222 }
225class RunConfigViewSet(mixins.CreateModelMixin,
226 mixins.ListModelMixin,
227 mixins.RetrieveModelMixin,
228 viewsets.GenericViewSet):
229 queryset = RunConfig.objects.all().order_by('-id')
230 serializer_class = RunConfigSerializer
231 lookup_value_regex = r"[\w.-]+"
232 filterset_class = RunConfigFilter
234 @extend_schema(
235 parameters=[
236 OpenApiParameter(
237 name="id",
238 description="A unique ID or name identifying this runconfig.",
239 location=OpenApiParameter.PATH,
240 )
241 ]
242 )
243 def retrieve(self, request, pk=None):
244 runconfig = self._get_runcfg(pk)
245 serializer = RunConfigSerializer(runconfig)
246 return Response(serializer.data)
248 @classmethod
249 def known_failures_serialized(cls, runcfg):
250 f = KnownFailure.objects.filter(result__ts_run__runconfig=runcfg)
251 failures = f.prefetch_related('result__status', 'result__test', 'result__test__testsuite',
252 'result__ts_run__machine', 'matched_ifa__issue__bugs',
253 'matched_ifa__issue__bugs__tracker')
254 return KnownIssuesSerializer(failures, read_only=True, many=True)
256 def _get_runcfg(self, obj_id):
257 return get_obj_by_id_or_name(RunConfig, obj_id)
259 @action(detail=True)
260 def known_failures(self, request, pk=None):
261 runcfg = self._get_runcfg(pk)
262 return Response(self.known_failures_serialized(runcfg).data, status=status.HTTP_200_OK)
264 @extend_schema(
265 description="Compare two runconfigs",
266 parameters=[
267 OpenApiParameter(
268 name="id",
269 description="Name or ID of of the RunConfig used for the test suite run",
270 location=OpenApiParameter.PATH
271 ),
272 OpenApiParameter(
273 name="to",
274 description="Id or name of RunConfig to compare",
275 required=True
276 ),
277 OpenApiParameter(
278 name="no_compress",
279 description="Should not compress comparison results",
280 type=bool
281 ),
282 OpenApiParameter(
283 name="summary",
284 description="Return raw text summary (to enable add this parameter to request)",
285 )
286 ],
287 responses={200: RunConfigDiffSerializer}
288 )
289 @action(detail=True)
290 def compare(self, request, pk=None):
291 runcfg_from = self._get_runcfg(pk)
292 runcfg_to = self._get_runcfg(request.GET.get('to'))
293 no_compress = request.GET.get('no_compress') is not None
295 diff = runcfg_from.compare(runcfg_to, no_compress=no_compress)
296 if request.GET.get('summary') is None:
297 serializer = RunConfigDiffSerializer(diff)
298 return Response(serializer.data, status=status.HTTP_200_OK)
299 else:
300 return HttpResponse(diff.text)
302 @extend_schema(
303 description=(
304 "Create test suite run object"
305 ),
306 parameters=[
307 OpenApiParameter(
308 name="id",
309 description="Id or name of base RunConfig",
310 location=OpenApiParameter.PATH
311 ),
312 ],
313 request=ImportTestSuiteRunSerializer,
314 responses={201: None},
315 )
316 @action(detail=True, methods=["post"], url_path='testsuiterun')
317 def import_test_suite_run(self, request, pk=None):
318 try:
319 request.data["runconfig_name"] = self._get_runcfg(pk).name
320 serializer = ImportTestSuiteRunSerializer(data=request.data)
321 if serializer.is_valid():
322 serializer.save()
323 return Response(status=status.HTTP_200_OK)
324 else:
325 return Response({"message": serializer.errors}, status=status.HTTP_400_BAD_REQUEST)
326 except Exception as error:
327 return Response({"message": str(error)}, status=status.HTTP_400_BAD_REQUEST)
330class ComponentViewSet(viewsets.ReadOnlyModelViewSet):
331 queryset = Component.objects.all().order_by('-id')
332 serializer_class = ComponentSerializer
335class BuildViewSet(mixins.CreateModelMixin,
336 mixins.ListModelMixin,
337 mixins.RetrieveModelMixin,
338 mixins.UpdateModelMixin,
339 viewsets.GenericViewSet):
340 queryset = Build.objects.all().order_by('-id')
341 serializer_class = BuildSerializer
343 @extend_schema(
344 parameters=[
345 OpenApiParameter(
346 name="id",
347 description="A unique ID or name identifying this build.",
348 location=OpenApiParameter.PATH,
349 )
350 ]
351 )
352 def retrieve(self, request, pk=None):
353 build = get_obj_by_id_or_name(Build, pk)
354 serializer = BuildSerializer(build)
355 return Response(serializer.data)
358class MachineViewSet(viewsets.ViewSet, mixins.ListModelMixin, viewsets.GenericViewSet):
359 queryset = Machine.objects.all()
360 serializer_class = RestViewMachineSerializer
361 filterset_fields = {
362 "id": ["exact"],
363 "name": ["exact"],
364 "public": ["exact"],
365 "vetted_on": ["isnull"],
366 }
368 def create(self, request, *args, **kwargs):
369 serializer = ImportMachineSerializer(data=request.data)
370 try:
371 if serializer.is_valid():
372 machine = serializer.save()
373 return Response(
374 {"status": "success", "data": RestViewMachineSerializer(machine).data},
375 status=status.HTTP_201_CREATED
376 )
377 return Response({"status": "error", "message": serializer.errors}, status=status.HTTP_400_BAD_REQUEST)
378 except ImportMachineError as err:
379 return Response({"status": "error", "message": str(err)}, status=status.HTTP_400_BAD_REQUEST)
381 @action(detail=True, methods=["post"])
382 def vet(self, request, pk):
383 return object_vet(Machine, pk)
385 @action(detail=True, methods=["post"])
386 def suppress(self, request, pk):
387 return object_suppress(Machine, pk)
390class TestFilter(filters.FilterSet):
391 class Meta:
392 model = Test
393 fields = {
394 'id': ['exact'],
395 'name': ['exact'],
396 'testsuite': ['exact'],
397 'public': ['exact'],
398 'added_on': ['lte', 'gte'],
399 'vetted_on': ['isnull'],
400 }
401 filter_overrides = {
402 models.DateTimeField: {
403 'filter_class': django_filters.IsoDateTimeFilter
404 },
405 }
408class TestSet(viewsets.ReadOnlyModelViewSet):
409 queryset = Test.objects.all().order_by('-id')
410 serializer_class = TestSerializer
411 filterset_class = TestFilter
413 @action(detail=True, methods=["post"])
414 def vet(self, request, pk):
415 return object_vet(Test, pk)
417 @action(detail=True, methods=["post"])
418 def suppress(self, request, pk):
419 return object_suppress(Test, pk)
422class UnknownFailureViewSet(viewsets.ReadOnlyModelViewSet):
423 queryset = UnknownFailure.objects.all().order_by('-id')
424 serializer_class = UnknownFailureSerializer
425 pagination_class = CustomPagination
427 EXTRA_FIELDS_ARG = OpenApiParameter(
428 name="extra_fields",
429 description=(
430 f"Comma-separated list of fields to expand (available choices: "
431 f"{', '.join(serializer_class.extra_fields())})"
432 ),
433 type=str,
434 )
436 @extend_schema(description="Retrieve a single unknown failure", parameters=[EXTRA_FIELDS_ARG])
437 def retrieve(self, request, pk):
438 extra_fields = request.query_params.get('extra_fields', '').split(',')
439 failure = get_object_or_404(UnknownFailure, pk=pk)
440 serializer = self.serializer_class(failure, extra_fields=extra_fields)
441 return Response(serializer.data)
443 @extend_schema(description="List all unknown failures", parameters=[EXTRA_FIELDS_ARG])
444 def list(self, request):
445 page = self.paginate_queryset(self.queryset)
446 extra_fields = request.query_params.get('extra_fields', '').split(',')
447 serializer = self.serializer_class(page, extra_fields=extra_fields, many=True)
448 return self.get_paginated_response(serializer.data)
451class TextStatusViewSet(viewsets.ReadOnlyModelViewSet):
452 queryset = TextStatus.objects.all().order_by('-id')
453 serializer_class = TextStatusSerializer
455 @action(detail=True)
456 def vet(self, request, pk):
457 return object_vet(TextStatus, pk)
459 @action(detail=True)
460 def suppress(self, request, pk):
461 return object_suppress(TextStatus, pk)
464class BugTrackerViewSet(viewsets.ReadOnlyModelViewSet):
465 queryset = BugTracker.objects.all().order_by('-id')
466 serializer_class = BugTrackerSerializer
469class BugViewSet(RetrieveAPIView):
470 serializer_class = BugCompleteSerializer
471 queryset = Bug.objects.all()
473 def _get_bugtracker(self):
474 tracker = self.kwargs.get('tracker')
476 try:
477 return BugTracker.objects.get(pk=int(tracker))
478 except (ValueError, BugTracker.DoesNotExist):
479 pass
481 try:
482 return BugTracker.objects.get(name=tracker)
483 except BugTracker.DoesNotExist:
484 pass
486 return get_object_or_404(BugTracker, short_name=tracker)
488 def retrieve(self, request, *args, **kwargs):
489 bug = get_object_or_404(Bug, tracker=self._get_bugtracker(), bug_id=kwargs.get('bug_id'))
490 serializer = self.get_serializer(bug)
491 return Response(serializer.data)
494class BugTrackerAccountViewSet(viewsets.ModelViewSet):
495 # WARNING: we do not yet perform access control for setting who is a user or developer because of the limited damage
496 # this can cause and the annoyance of having to ask large group of users to authenticate then be granted the
497 # privilege to change the roles
498 permission_classes = []
499 authentication_classes = []
501 queryset = BugTrackerAccount.objects.all().order_by('-id')
502 serializer_class = BugTrackerAccountSerializer
504 http_method_names = ['get', 'patch']
507class ShortenerViewSet(ListCreateAPIView, viewsets.GenericViewSet):
508 # WARNING: No access control is performed because these objects can be created simply by navigating the website
509 permission_classes = []
510 authentication_classes = []
512 queryset = Shortener.objects.all().order_by('-id')
513 serializer_class = ShortenerSerializer
515 def create(self, request, *args, **kwargs):
516 if request.method != "POST" or request.content_type != "application/json":
517 raise ValueError("Only JSON POST requests are supported")
519 data = json.loads(request.body)
520 fulls = data.get('full')
521 if fulls is None:
522 raise ValueError("Missing the field 'full' which should contain the full text to be shortened")
524 if isinstance(fulls, list):
525 shorts = [Shortener.get_or_create(full=f) for f in fulls]
526 serializer = self.get_serializer(shorts, many=True)
527 else:
528 short = Shortener.get_or_create(full=fulls)
529 serializer = self.get_serializer(short)
531 return JsonResponse(serializer.data, safe=False)
534@api_view()
535@permission_classes((permissions.AllowAny,))
536def metrics_passrate_per_runconfig_view(request):
537 user_query = QueryCreator(request, TestResult).request_to_query()
538 history = MetricPassRatePerRunconfig(user_query)
539 return Response(serialize_MetricPassRatePerRunconfig(history))
542@api_view(['GET', 'POST'])
543@permission_classes((permissions.AllowAny,))
544def metrics_passrate_per_test_view(request):
545 user_query = QueryCreator(request, TestResult).request_to_query()
546 passrate = MetricPassRatePerTest(user_query)
547 return Response(serialize_MetricPassRatePerTest(passrate))