Coverage for CIResults/rest_views.py: 100%
288 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-23 13:11 +0000
1from django.shortcuts import get_object_or_404
2from django.http import HttpResponse, JsonResponse
3from django.db import transaction, models
4import django_filters
6from rest_framework.decorators import action, api_view, permission_classes
7from rest_framework.generics import RetrieveAPIView, ListCreateAPIView
8from rest_framework.response import Response
9from rest_framework.pagination import PageNumberPagination
10from rest_framework import status, viewsets, permissions, mixins
11from django_filters import rest_framework as filters
12from drf_spectacular.utils import extend_schema, OpenApiParameter
14from .serializers import ImportTestSuiteRunSerializer, IssueFilterSerializer, RunConfigSerializer
15from .serializers import BuildSerializer, TestSerializer, BugTrackerAccountSerializer, KnownIssuesSerializer
16from .serializers import serialize_MetricPassRatePerRunconfig, serialize_MetricPassRatePerTest, BugTrackerSerializer
17from .serializers import BugCompleteSerializer, ShortenerSerializer, RestViewMachineSerializer, ComponentSerializer
18from .serializers import ImportMachineError, ImportMachineSerializer, RunConfigDiffSerializer, RestIssueSerializer
19from .filtering import QueryCreator
21from .models import Component, Build, Test, Machine, RunConfigTag, RunConfig, TextStatus, Bug, TestResult
22from .models import IssueFilter, Issue, KnownFailure, MachineTag, BugTrackerAccount, BugTracker
23from .metrics import MetricPassRatePerRunconfig, MetricPassRatePerTest
25from shortener.models import Shortener
27import json
28import re
31def get_obj_by_id_or_name(model, key):
32 try:
33 return model.objects.get(pk=int(key))
34 except ValueError:
35 pass
37 return get_object_or_404(model, name=key)
40class CustomPagination(PageNumberPagination):
41 page_size = 100
42 page_size_query_param = 'page_size'
43 max_page_size = None
45 def get_page_size(self, request):
46 if self.page_size_query_param:
47 try:
48 page_size = int(request.query_params.get(self.page_size_query_param, self.page_size))
49 except ValueError:
50 return self.page_size
52 # Clamp the maximum size
53 if self.max_page_size:
54 page_size = min(page_size, self.max_page_size)
56 # Do not set any limits if the page size is null or negative
57 if page_size <= 0:
58 return None
59 else:
60 return page_size
62 return self.page_size
65class IssueViewSet(viewsets.ReadOnlyModelViewSet):
66 queryset = Issue.objects.all().order_by('-id')
67 serializer_class = RestIssueSerializer
69 def patch(self, request, pk):
70 issue = Issue.objects.get(pk=pk)
71 serializer = RestIssueSerializer(issue, data=request.data, partial=True)
72 if serializer.is_valid():
73 serializer.save()
74 return Response(data=serializer.data, status=status.HTTP_200_OK)
75 return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST)
77 @classmethod
78 def _execute_action(cls, request, pk, permsission_name, action):
79 if not request.user.has_perm(permsission_name):
80 return Response(data={"message": f"User {request.user} doesn't have sufficient permissions"},
81 status=status.HTTP_401_UNAUTHORIZED)
82 try:
83 issue = get_object_or_404(Issue, pk=pk)
84 action(issue)
85 except Exception as err:
86 return Response(data={"message": str(err)}, status=status.HTTP_400_BAD_REQUEST)
87 return Response(status=status.HTTP_200_OK)
89 @action(detail=True)
90 def archive(self, request, pk=None):
91 return self._execute_action(request, pk, "CIResults.archive_issue", lambda issue: issue.archive(request.user))
93 @action(detail=True)
94 def restore(self, request, pk=None):
95 return self._execute_action(request, pk, "CIResults.restore_issue", lambda issue: issue.restore())
98class IssueFilterViewSet(viewsets.ModelViewSet):
99 queryset = IssueFilter.objects.all().order_by('-id').prefetch_related('tags', 'tests__testsuite',
100 'tests__first_runconfig',
101 'machine_tags', 'machines',
102 'statuses__testsuite')
103 serializer_class = IssueFilterSerializer
104 pagination_class = CustomPagination
106 def __check_list__(self, request_data, field, field_name, db_class, errors):
107 objects = set(request_data.get(field, []))
109 objects_db = dict()
110 for obj in db_class.objects.filter(id__in=objects):
111 objects_db[obj.id] = obj
113 if len(objects) != len(objects_db):
114 errors.append("At least one {} does not exist".format(field_name))
116 return objects, objects_db
118 def __get_or_None__(self, klass, field, request_dict, errors):
119 obj = None
120 if field in request_dict:
121 obj_id = request_dict[field]
123 # Do not consider empty strings as meaning a valid value
124 if isinstance(obj_id, str) and len(obj_id) == 0:
125 return None
127 # Convert the id to an int or fail
128 try:
129 obj_id = int(obj_id)
130 except Exception:
131 errors.append("The field '{}' needs to be an integer".format(field))
132 return None
134 # Try getting the object
135 obj = klass.objects.filter(id=obj_id).first()
136 if obj is None:
137 errors.append("The object referenced by '{}' does not exist".format(field))
139 return obj
141 def get_queryset(self):
142 queryset = self.queryset
143 if description := self.request.query_params.get("description"):
144 queryset = queryset.filter(description__contains=description)
145 return queryset
147 @transaction.atomic
148 def create(self, request):
149 errors = []
150 if len(request.data.get('description', '')) == 0:
151 errors.append("The field 'description' cannot be empty")
153 # Check if the filter should replace another one
154 edit_filter = self.__get_or_None__(IssueFilter, 'edit_filter',
155 request.data, errors)
156 edit_issue = self.__get_or_None__(Issue, 'edit_issue',
157 request.data, errors)
159 # Check that all the tags, machines, tests, and statuses are present
160 tags, tags_db = self.__check_list__(request.data, "tags", "tag", RunConfigTag, errors)
161 machine_tags, machine_tags_db = self.__check_list__(request.data, "machine_tags", "machine tag",
162 MachineTag, errors)
163 machines, machines_db = self.__check_list__(request.data, "machines", "machine", Machine, errors)
164 tests, tests_db = self.__check_list__(request.data, "tests", "test", Test, errors)
165 statuses, statuses_db = self.__check_list__(request.data, "statuses", "status", TextStatus, errors)
167 # Check the regular expressions
168 for field in ['stdout_regex', 'stderr_regex', 'dmesg_regex']:
169 try:
170 re.compile(request.data.get(field, ""))
171 except Exception:
172 errors.append("The field '{}' does not contain a valid regular expression".format(field))
174 # Create the object or fail depending on whether we got errors or not
175 if len(errors) == 0:
176 filter = IssueFilter.objects.create(description=request.data.get('description'),
177 stdout_regex=request.data.get('stdout_regex', ""),
178 stderr_regex=request.data.get('stderr_regex', ""),
179 dmesg_regex=request.data.get('dmesg_regex', ""),
180 user_query=request.data.get('user_query', ""))
182 filter.tags.add(*tags_db)
183 filter.machines.add(*machines_db)
184 filter.machine_tags.add(*machine_tags_db)
185 filter.tests.add(*tests_db)
186 filter.statuses.add(*statuses_db)
188 # If this filter is supposed to replace another filter
189 if edit_filter is not None:
190 if edit_issue is not None:
191 edit_issue.replace_filter(edit_filter, filter, request.user)
192 else:
193 edit_filter.replace(filter, request.user)
195 serializer = IssueFilterSerializer(filter)
196 return Response(serializer.data, status=status.HTTP_201_CREATED)
197 else:
198 return Response(errors, status=status.HTTP_400_BAD_REQUEST)
201class RunConfigFilter(filters.FilterSet):
202 class Meta:
203 model = RunConfig
204 fields = {
205 "name": ["exact", "contains"],
206 "builds__name": ["exact", "contains"],
207 }
210class RunConfigViewSet(mixins.CreateModelMixin,
211 mixins.ListModelMixin,
212 mixins.RetrieveModelMixin,
213 viewsets.GenericViewSet):
214 queryset = RunConfig.objects.all().order_by('-id')
215 serializer_class = RunConfigSerializer
216 lookup_value_regex = r"[\w.-]+"
217 filterset_class = RunConfigFilter
219 @extend_schema(
220 parameters=[
221 OpenApiParameter(
222 name="id",
223 description="A unique ID or name identifying this runconfig.",
224 location=OpenApiParameter.PATH,
225 )
226 ]
227 )
228 def retrieve(self, request, pk=None):
229 runconfig = self._get_runcfg(pk)
230 serializer = RunConfigSerializer(runconfig)
231 return Response(serializer.data)
233 @classmethod
234 def known_failures_serialized(cls, runcfg):
235 f = KnownFailure.objects.filter(result__ts_run__runconfig=runcfg)
236 failures = f.prefetch_related('result__status', 'result__test', 'result__test__testsuite',
237 'result__ts_run__machine', 'matched_ifa__issue__bugs',
238 'matched_ifa__issue__bugs__tracker')
239 return KnownIssuesSerializer(failures, read_only=True, many=True)
241 def _get_runcfg(self, obj_id):
242 return get_obj_by_id_or_name(RunConfig, obj_id)
244 @action(detail=True)
245 def known_failures(self, request, pk=None):
246 runcfg = self._get_runcfg(pk)
247 return Response(self.known_failures_serialized(runcfg).data, status=status.HTTP_200_OK)
249 @extend_schema(
250 description="Compare two runconfigs",
251 parameters=[
252 OpenApiParameter(
253 name="id",
254 description="Name or ID of of the RunConfig used for the test suite run",
255 location=OpenApiParameter.PATH
256 ),
257 OpenApiParameter(
258 name="to",
259 description="Id or name of RunConfig to compare",
260 required=True
261 ),
262 OpenApiParameter(
263 name="no_compress",
264 description="Should not compress comparison results",
265 type=bool
266 ),
267 OpenApiParameter(
268 name="summary",
269 description="Return raw text summary (to enable add this parameter to request)",
270 )
271 ],
272 responses={200: RunConfigDiffSerializer}
273 )
274 @action(detail=True)
275 def compare(self, request, pk=None):
276 runcfg_from = self._get_runcfg(pk)
277 runcfg_to = self._get_runcfg(request.GET.get('to'))
278 no_compress = request.GET.get('no_compress') is not None
280 diff = runcfg_from.compare(runcfg_to, no_compress=no_compress)
281 if request.GET.get('summary') is None:
282 serializer = RunConfigDiffSerializer(diff)
283 return Response(serializer.data, status=status.HTTP_200_OK)
284 else:
285 return HttpResponse(diff.text)
287 @extend_schema(
288 description=(
289 "Create test suite run object"
290 ),
291 parameters=[
292 OpenApiParameter(
293 name="id",
294 description="Id or name of base RunConfig",
295 location=OpenApiParameter.PATH
296 ),
297 ],
298 request=ImportTestSuiteRunSerializer,
299 responses={201: None},
300 )
301 @action(detail=True, methods=["post"], url_path='testsuiterun')
302 def import_test_suite_run(self, request, pk=None):
303 try:
304 request.data["runconfig_name"] = self._get_runcfg(pk).name
305 serializer = ImportTestSuiteRunSerializer(data=request.data)
306 if serializer.is_valid():
307 serializer.save()
308 return Response(status=status.HTTP_200_OK)
309 else:
310 return Response({"message": serializer.errors}, status=status.HTTP_400_BAD_REQUEST)
311 except Exception as error:
312 return Response({"message": str(error)}, status=status.HTTP_400_BAD_REQUEST)
315class ComponentViewSet(viewsets.ReadOnlyModelViewSet):
316 queryset = Component.objects.all().order_by('-id')
317 serializer_class = ComponentSerializer
320class BuildViewSet(mixins.CreateModelMixin,
321 mixins.ListModelMixin,
322 mixins.RetrieveModelMixin,
323 mixins.UpdateModelMixin,
324 viewsets.GenericViewSet):
325 queryset = Build.objects.all().order_by('-id')
326 serializer_class = BuildSerializer
328 @extend_schema(
329 parameters=[
330 OpenApiParameter(
331 name="id",
332 description="A unique ID or name identifying this build.",
333 location=OpenApiParameter.PATH,
334 )
335 ]
336 )
337 def retrieve(self, request, pk=None):
338 build = get_obj_by_id_or_name(Build, pk)
339 serializer = BuildSerializer(build)
340 return Response(serializer.data)
343class MachineViewSet(viewsets.ViewSet, mixins.ListModelMixin, viewsets.GenericViewSet):
344 queryset = Machine.objects.all()
345 serializer_class = RestViewMachineSerializer
347 def create(self, request, *args, **kwargs):
348 serializer = ImportMachineSerializer(data=request.data)
349 try:
350 if serializer.is_valid():
351 machine = serializer.save()
352 return Response(
353 {"status": "success", "data": RestViewMachineSerializer(machine).data},
354 status=status.HTTP_201_CREATED
355 )
356 return Response({"status": "error", "message": serializer.errors}, status=status.HTTP_400_BAD_REQUEST)
357 except ImportMachineError as err:
358 return Response({"status": "error", "message": str(err)}, status=status.HTTP_400_BAD_REQUEST)
361class TestFilter(filters.FilterSet):
362 class Meta:
363 model = Test
364 fields = {
365 'id': ['exact'],
366 'name': ['exact'],
367 'testsuite': ['exact'],
368 'public': ['exact'],
369 'added_on': ['lte', 'gte'],
370 'vetted_on': ['isnull'],
371 }
372 filter_overrides = {
373 models.DateTimeField: {
374 'filter_class': django_filters.IsoDateTimeFilter
375 },
376 }
379class TestSet(viewsets.ReadOnlyModelViewSet):
380 queryset = Test.objects.all().order_by('-id')
381 serializer_class = TestSerializer
382 filterset_fields = ('id', 'testsuite', 'public', 'vetted_on', 'name')
383 filter_class = TestFilter
385 @action(detail=True)
386 def vet(self, request, pk):
387 if not request.user.has_perm("CIResults.vet_test"):
388 return Response({"message": f"User {request.user} doesn't have sufficient permissions"},
389 status=status.HTTP_403_FORBIDDEN)
390 test = get_object_or_404(Test, pk=pk)
391 if not test.vetted:
392 test.vet()
393 return Response(status=status.HTTP_200_OK)
395 @action(detail=True)
396 def suppress(self, request, pk):
397 if not request.user.has_perm("CIResults.suppress_test"):
398 return Response({"message": f"User {request.user} doesn't have sufficient permissions"},
399 status=status.HTTP_403_FORBIDDEN)
400 test = get_object_or_404(Test, pk=pk)
401 if test.vetted:
402 test.suppress()
403 return Response(status=status.HTTP_200_OK)
406class BugTrackerViewSet(viewsets.ReadOnlyModelViewSet):
407 queryset = BugTracker.objects.all().order_by('-id')
408 serializer_class = BugTrackerSerializer
411class BugViewSet(RetrieveAPIView):
412 serializer_class = BugCompleteSerializer
413 queryset = Bug.objects.all()
415 def _get_bugtracker(self):
416 tracker = self.kwargs.get('tracker')
418 try:
419 return BugTracker.objects.get(pk=int(tracker))
420 except (ValueError, BugTracker.DoesNotExist):
421 pass
423 try:
424 return BugTracker.objects.get(name=tracker)
425 except BugTracker.DoesNotExist:
426 pass
428 return get_object_or_404(BugTracker, short_name=tracker)
430 def retrieve(self, request, *args, **kwargs):
431 bug = get_object_or_404(Bug, tracker=self._get_bugtracker(), bug_id=kwargs.get('bug_id'))
432 serializer = self.get_serializer(bug)
433 return Response(serializer.data)
436class BugTrackerAccountViewSet(viewsets.ModelViewSet):
437 # WARNING: we do not yet perform access control for setting who is a user or developer because of the limited damage
438 # this can cause and the annoyance of having to ask large group of users to authenticate then be granted the
439 # privilege to change the roles
440 permission_classes = []
441 authentication_classes = []
443 queryset = BugTrackerAccount.objects.all().order_by('-id')
444 serializer_class = BugTrackerAccountSerializer
446 http_method_names = ['get', 'patch']
449class ShortenerViewSet(ListCreateAPIView, viewsets.GenericViewSet):
450 # WARNING: No access control is performed because these objects can be created simply by navigating the website
451 permission_classes = []
452 authentication_classes = []
454 queryset = Shortener.objects.all().order_by('-id')
455 serializer_class = ShortenerSerializer
457 def create(self, request, *args, **kwargs):
458 if request.method != "POST" or request.content_type != "application/json":
459 raise ValueError("Only JSON POST requests are supported")
461 data = json.loads(request.body)
462 fulls = data.get('full')
463 if fulls is None:
464 raise ValueError("Missing the field 'full' which should contain the full text to be shortened")
466 if isinstance(fulls, list):
467 shorts = [Shortener.get_or_create(full=f) for f in fulls]
468 serializer = self.get_serializer(shorts, many=True)
469 else:
470 short = Shortener.get_or_create(full=fulls)
471 serializer = self.get_serializer(short)
473 return JsonResponse(serializer.data, safe=False)
476@api_view()
477@permission_classes((permissions.AllowAny,))
478def metrics_passrate_per_runconfig_view(request):
479 user_query = QueryCreator(request, TestResult).request_to_query()
480 history = MetricPassRatePerRunconfig(user_query)
481 return Response(serialize_MetricPassRatePerRunconfig(history))
484@api_view(['GET', 'POST'])
485@permission_classes((permissions.AllowAny,))
486def metrics_passrate_per_test_view(request):
487 user_query = QueryCreator(request, TestResult).request_to_query()
488 passrate = MetricPassRatePerTest(user_query)
489 return Response(serialize_MetricPassRatePerTest(passrate))