feat(api): add compliance provider filters#11587
Open
AdriiiPRodri wants to merge 1 commit into
Open
Conversation
- Add latest scan provider filtering to compliance overviews - Support provider ID, type, and group filters with multi-value variants - Cover provider filter behavior with API tests
|
Important Review skippedAuto reviews are disabled on base/target branches other than the default branch. Please check the settings in the CodeRabbit UI or the ⚙️ Run configurationConfiguration used: Organization UI Review profile: ASSERTIVE Plan: Pro Plus Run ID: You can disable this status message by setting the Use the checkbox below for a quick retry:
✨ Finishing Touches🧪 Generate unit tests (beta)
Comment |
josema-xyz
reviewed
Jun 15, 2026
josema-xyz
left a comment
josema-xyz
left a comment
Contributor
There was a problem hiding this comment.
Two comments, also, no CHANGELOG.
Comment on lines
+4767
to
+4911
| def _normalize_jsonapi_params(self, query_params, exclude_keys=None): | ||
| """Convert JSON:API filter params into django-filter keys.""" | ||
| exclude_keys = exclude_keys or set() | ||
| normalized = QueryDict(mutable=True) | ||
| for key, values in query_params.lists(): | ||
| normalized_key = ( | ||
| key[7:-1] if key.startswith("filter[") and key.endswith("]") else key | ||
| ) | ||
| normalized_key = normalized_key.replace(".", "__") | ||
| if normalized_key not in exclude_keys: | ||
| normalized.setlist(normalized_key, values) | ||
| return normalized | ||
|
|
||
| def _apply_compliance_filterset(self, queryset, exclude_keys=None): | ||
| normalized_params = self._normalize_jsonapi_params( | ||
| self.request.query_params, | ||
| exclude_keys=set(exclude_keys or []), | ||
| ) | ||
| filterset = self.filterset_class(normalized_params, queryset=queryset) | ||
| if not filterset.is_valid(): | ||
| raise ValidationError(filterset.errors) | ||
| return filterset.qs | ||
|
|
||
| def _csv_filter_values(self, value): | ||
| return [item.strip() for item in value.split(",") if item.strip()] | ||
|
|
||
| def _validate_uuid_filter_values(self, field_name, values): | ||
| try: | ||
| for value in values: | ||
| uuid.UUID(str(value)) | ||
| except (TypeError, ValueError, AttributeError): | ||
| raise ValidationError({field_name: ["Enter a valid UUID."]}) | ||
|
|
||
| def _has_provider_filters(self): | ||
| return any( | ||
| self.request.query_params.get(f"filter[{key}]") | ||
| for key in self.PROVIDER_FILTER_QUERY_KEYS | ||
| ) | ||
|
|
||
| def _validate_scan_selection(self, scan_id, has_provider_filters): | ||
| if scan_id and has_provider_filters: | ||
| raise ValidationError( | ||
| [ | ||
| { | ||
| "detail": "Use either filter[scan_id] or provider filters.", | ||
| "status": 400, | ||
| "source": {"pointer": "filter[scan_id]"}, | ||
| "code": "invalid", | ||
| } | ||
| ] | ||
| ) | ||
|
|
||
| if scan_id: | ||
| self._validate_uuid_filter_values("scan_id", [scan_id]) | ||
| return | ||
|
|
||
| if has_provider_filters: | ||
| return | ||
|
|
||
| raise ValidationError( | ||
| [ | ||
| { | ||
| "detail": "This query parameter is required unless a provider filter is provided.", | ||
| "status": 400, | ||
| "source": {"pointer": "filter[scan_id]"}, | ||
| "code": "required", | ||
| } | ||
| ] | ||
| ) | ||
|
|
||
| def _extract_provider_filters_from_params(self): | ||
| """Extract provider filters for the Scan queryset.""" | ||
| params = self.request.query_params | ||
| filters = {} | ||
| valid_provider_types = { | ||
| choice[0] for choice in Provider.ProviderChoices.choices | ||
| } | ||
|
|
||
| provider_id = params.get("filter[provider_id]") | ||
| if provider_id: | ||
| self._validate_uuid_filter_values("provider_id", [provider_id]) | ||
| filters["provider_id"] = provider_id | ||
|
|
||
| provider_id_in = params.get("filter[provider_id__in]") or params.get( | ||
| "filter[provider_id.in]" | ||
| ) | ||
| if provider_id_in: | ||
| values = self._csv_filter_values(provider_id_in) | ||
| self._validate_uuid_filter_values("provider_id__in", values) | ||
| filters["provider_id__in"] = values | ||
|
|
||
| provider_type = params.get("filter[provider_type]") | ||
| if provider_type: | ||
| if provider_type not in valid_provider_types: | ||
| raise ValidationError( | ||
| {"provider_type": f"Invalid choice: {provider_type}"} | ||
| ) | ||
| filters["provider__provider"] = provider_type | ||
|
|
||
| provider_type_in = params.get("filter[provider_type__in]") or params.get( | ||
| "filter[provider_type.in]" | ||
| ) | ||
| if provider_type_in: | ||
| values = self._csv_filter_values(provider_type_in) | ||
| invalid = [value for value in values if value not in valid_provider_types] | ||
| if invalid: | ||
| raise ValidationError( | ||
| {"provider_type__in": f"Invalid choices: {', '.join(invalid)}"} | ||
| ) | ||
| filters["provider__provider__in"] = values | ||
|
|
||
| provider_groups = params.get("filter[provider_groups]") | ||
| if provider_groups: | ||
| self._validate_uuid_filter_values("provider_groups", [provider_groups]) | ||
| filters["provider__provider_groups__id"] = provider_groups | ||
|
|
||
| provider_groups_in = params.get("filter[provider_groups__in]") or params.get( | ||
| "filter[provider_groups.in]" | ||
| ) | ||
| if provider_groups_in: | ||
| values = self._csv_filter_values(provider_groups_in) | ||
| self._validate_uuid_filter_values("provider_groups__in", values) | ||
| filters["provider__provider_groups__id__in"] = values | ||
|
|
||
| return filters | ||
|
|
||
| def _latest_scan_ids_for_provider_filters(self): | ||
| role = get_role(self.request.user, self.request.tenant_id) | ||
| scans = Scan.all_objects.filter( | ||
| tenant_id=self.request.tenant_id, | ||
| state=StateChoices.COMPLETED, | ||
| ) | ||
|
|
||
| if not getattr(role, Permissions.UNLIMITED_VISIBILITY.value, False): | ||
| scans = scans.filter(provider__in=get_providers(role)) | ||
|
|
||
| provider_filters = self._extract_provider_filters_from_params() | ||
| if provider_filters: | ||
| scans = scans.filter(**provider_filters) | ||
|
|
||
| return ( | ||
| scans.order_by("provider_id", "-inserted_at") | ||
| .distinct("provider_id") | ||
| .values_list("id", flat=True) | ||
| ) |
Contributor
There was a problem hiding this comment.
These helpers are almost identical to ones already in OverviewViewSet and FindingGroupViewSet. That's a third copy. Could they live in one shared place instead? They've already started to drift apart.
|
|
||
|
|
||
| class ComplianceOverviewFilter(FilterSet): | ||
| class ComplianceOverviewFilter(BaseScanProviderFilter): |
Contributor
There was a problem hiding this comment.
This class inherits the provider filters, but the viewset filters providers by hand and skips them. They still show up in the API docs, but they're not what does the filtering. Is keeping both on purpose? It's hard to tell which one is the real one.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Context
Compliance overview consumers can currently request data by
filter[scan_id]. This PR adds provider-scoped filtering to the compliance overview endpoints so callers can request the latest completed scan per matching provider, consistent with the provider filtering added in the parent branch.Description
provider_id,provider_id__in,provider_type,provider_type__in,provider_groups, andprovider_groups__infilters to compliance overviews.filter[scan_id]behavior for scan-scoped callers.Steps to review
cd api && uv run pytest src/backend/api/tests/test_views.py::TestComplianceOverviewViewSet -q.cd api && uv run ruff format --check src/backend/api/filters.py src/backend/api/v1/views.py src/backend/api/tests/test_views.py && uv run ruff check src/backend/api/filters.py src/backend/api/v1/views.py src/backend/api/tests/test_views.py./api/v1/compliance-overviewswithfilter[provider_id],filter[provider_id__in],filter[provider_type],filter[provider_groups], andfilter[provider_groups__in]./api/v1/docs#tag/Compliance-Overview/operation/api_v1_compliance_overviews_listshows the new provider filters.Checklist
Community Checklist
SDK/CLI
UI
API
API verification performed locally:
uv run pytest src/backend/api/tests/test_views.py::TestComplianceOverviewViewSet -q: 30 passed, 1 xpassed.uv run ruff format --check src/backend/api/filters.py src/backend/api/v1/views.py src/backend/api/tests/test_views.py && uv run ruff check src/backend/api/filters.py src/backend/api/v1/views.py src/backend/api/tests/test_views.py: passed./api/v1/docs#tag/Compliance-Overview/operation/api_v1_compliance_overviews_list.Performance details:
ORDER BY provider_id, -inserted_atandDISTINCT ON (provider_id), then filters compliance rows byscan_id__in. Provider filters are excluded from the compliance filterset after scan resolution so the large compliance table does not repeat provider joins.scans_prov_ins_desc_idxfor latest completed scan lookup andcro_scan_comp_reg_idxfor compliance overview rows.GET /api/v1/compliance-overviews?filter[provider_type]=aws: median30.82 ms, max34.84 ms.GET /api/v1/compliance-overviews?filter[provider_groups__in]=...: median25.26 ms, max31.24 ms.GET /api/v1/compliance-overviews/requirements?filter[provider_id__in]=...&filter[compliance_id]=cis_1.4_aws: median24.16 ms, max31.90 ms.EXPLAIN ANALYZEfor the latest-scan compliance aggregation completed in0.504 ms, with82shared buffer hits and no sequential scan oncompliance_requirements_overviews.License
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.