Skip to content
This repository was archived by the owner on Oct 24, 2024. It is now read-only.

Commit 71cf475

Browse files
author
Søren Howe Gersager
committed
Merge branch 'hotfix/3.7.1' into 'master'
Hotfix/3.7.1 - master See merge request bevillingsplatform/bevillingsplatform!1194
2 parents 580480b + 5202556 commit 71cf475

File tree

14 files changed

+857
-273
lines changed

14 files changed

+857
-273
lines changed

‎NEWS.rst

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,15 @@
1+
Version 3.7.1, 2022-01-24
2+
-------------------------
3+
4+
Hotfix release
5+
6+
Bug fixes
7+
^^^^^^^^^
8+
* Use appropriation_date for determining dst_report_type for Appropriations.
9+
* Use fromDate and toDate for DST in the Appropriations API.
10+
* Various fixes for the way we find DST duplicate/consolidated Appropriations.
11+
* Extract and use DST start/end date logic for duplicate/consolidated Appropriations.
12+
113
Version 3.7.0, 2022-01-10
214
-------------------------
315

‎VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
3.7.0
1+
3.7.1

‎backend/core/filters.py

Lines changed: 1 addition & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,13 @@
1010
Filters allow us to do basic search for objects on allowed field without
1111
adding the complexity of an entire search engine nor of custom queries.
1212
"""
13-
import json
14-
1513
from django.utils import timezone
16-
from datetime import date, datetime, time
14+
from datetime import date
1715
from dateutil.relativedelta import relativedelta, MO, SU
1816

19-
from django.forms import DateField, Field, ValidationError
2017
from django.utils.translation import gettext
2118

2219
import django_filters as filters
23-
from django_filters.utils import handle_timezone
2420

2521
from core.models import (
2622
Case,
@@ -30,68 +26,6 @@
3026
)
3127

3228

33-
class DateRangeField(Field):
34-
"""Custom DateRangeField for use in DateFromToRangeFilter."""
35-
36-
none_value = "None"
37-
38-
def compress(self, data_list):
39-
"""Override compress to convert dates."""
40-
if data_list:
41-
start_date, stop_date = data_list
42-
if start_date:
43-
start_date = handle_timezone(
44-
datetime.combine(start_date, time.min)
45-
)
46-
if stop_date:
47-
stop_date = handle_timezone(
48-
datetime.combine(stop_date, time.max)
49-
)
50-
return slice(start_date, stop_date)
51-
return None
52-
53-
def clean(self, value):
54-
"""Override clean to enforce "None" or dates."""
55-
if value:
56-
clean_data = []
57-
values = json.loads(value)
58-
if isinstance(values, (list, tuple)):
59-
for field_value in values:
60-
if field_value == self.none_value:
61-
clean_data.append(None)
62-
else:
63-
clean_data.append(DateField().clean(field_value))
64-
else:
65-
raise ValidationError(
66-
gettext(
67-
"DateRangeField skal være en liste"
68-
' indeholdende datoer eller "None"'
69-
)
70-
)
71-
return self.compress(clean_data)
72-
else:
73-
return self.compress([])
74-
75-
76-
class CustomDateFromToRangeFilter(filters.RangeFilter):
77-
r"""
78-
Custom DateFromToRangeFilter.
79-
80-
Works with our GraphQL API
81-
(and DRF) and takes a json list of date strings.
82-
83-
Sadly graphene-django does not support filters.DateFromToRangeFilter.
84-
https://github.com/graphql-python/graphene-django/issues/92
85-
86-
Example in GraphQL:
87-
dstDate:"[\"2016-12-01\",\"2016-12-31\"]"
88-
or
89-
dstDate:"[\"None\",\"2016-12-31\"]"
90-
"""
91-
92-
field_class = DateRangeField
93-
94-
9529
class CaseFilter(filters.FilterSet):
9630
"""Filter cases on the "expired" field."""
9731

@@ -134,17 +68,6 @@ class AppropriationFilter(filters.FilterSet):
13468
label=gettext("Team for Sagsbehandler for Sag")
13569
)
13670

137-
dst_date = CustomDateFromToRangeFilter(
138-
method="filter_dst_date",
139-
label=gettext(
140-
'DST dato fra/til (f.eks. "[\\"2021-01-02\\",\\"None\\"]")'
141-
),
142-
)
143-
144-
def filter_dst_date(self, queryset, name, value):
145-
"""Filter on DST date ("from_date" and "to_date")."""
146-
return queryset.appropriations_for_dst_payload(value.start, value.stop)
147-
14871
class Meta:
14972
model = Appropriation
15073
fields = "__all__"

‎backend/core/managers.py

Lines changed: 135 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
Sum,
1616
CharField,
1717
DecimalField,
18+
IntegerField,
1819
Func,
1920
Value,
2021
Q,
@@ -248,111 +249,171 @@ def annotate_main_activity_details_id(self):
248249
)
249250
)
250251

251-
def appropriations_for_dst_payload(self, from_date=None, to_date=None):
252+
def appropriations_for_dst_payload(
253+
self, from_date=None, to_date=None, initial_load=False
254+
):
252255
"""Filter appropriations for a Danmarks Statistik payload.
253256
254-
We annotate a report_type based on whether an Appropriation
255-
has "changed" or not which is based on:
256-
- changed acting municipality .
257-
- appropriation_date for activities
258-
259-
appropriation contains only granted main activities appropriated
260-
before 'from_date'
261-
-> exclude (as they have been included in a previous payload)
262-
263-
appropriation contains only granted main activities appropriated
264-
after 'from_date'
265-
-> status NEW
266-
267-
appropriation contains granted main activities appropriated both
268-
before and after 'from_date'
269-
-> status CHANGED
257+
This involves filtering and annotating an appropriate dst_report_type.
270258
"""
271259
from core.models import (
272260
MAIN_ACTIVITY,
273261
STATUS_GRANTED,
274262
Case as CaseModel,
263+
Activity,
275264
)
276265

277-
queryset = self.filter(
278-
activities__status=STATUS_GRANTED,
279-
activities__activity_type=MAIN_ACTIVITY,
280-
)
281-
# If to_date is set we cut off appropriations
282-
# with activities with a newer appropriation_date.
283-
if to_date:
284-
queryset = queryset.filter(
285-
activities__appropriation_date__lte=to_date
286-
)
266+
if not from_date:
267+
from_date = datetime.date(year=1970, month=1, day=1)
268+
if not to_date:
269+
to_date = timezone.now().date()
287270

288271
report_types = {
289272
"NEW": "Ny",
290273
"CHANGED": "Ændring",
291274
"CANCELLED": "Annullering",
292275
}
293276

294-
if from_date:
277+
queryset = self
278+
279+
if initial_load:
280+
# Full/initial load where all appropriations are marked "NEW".
281+
282+
# Cut off appropriations appropriated outside the
283+
# from_date->to_date range.
284+
queryset = (
285+
queryset.filter(
286+
activities__status=STATUS_GRANTED,
287+
activities__activity_type=MAIN_ACTIVITY,
288+
activities__appropriation_date__gte=from_date,
289+
activities__appropriation_date__lte=to_date,
290+
)
291+
.annotate(
292+
dst_report_type=Value(
293+
report_types["NEW"], output_field=CharField()
294+
)
295+
)
296+
.distinct()
297+
)
298+
299+
# Delta load where appropriations are marked "NEW" and "CHANGED"
300+
# based on different criteria.
301+
else:
295302
cases = CaseModel.objects.filter(appropriations__in=queryset)
296303
changed_cases = cases.filter_changed_cases_for_dst_payload(
297304
from_date, to_date
298305
)
299-
300-
main_activities_q = Q(activities__activity_type=MAIN_ACTIVITY)
301-
main_activities_appropriated_after_from_date_q = Q(
306+
# Cut off appropriations appropriated outside the
307+
# from_date->to_date range.
308+
queryset = queryset.filter(
309+
Q(
310+
Q(activities__appropriation_date__gte=from_date)
311+
& Q(activities__appropriation_date__lte=to_date)
312+
)
313+
| Q(case__in=changed_cases),
314+
activities__status=STATUS_GRANTED,
302315
activities__activity_type=MAIN_ACTIVITY,
303-
activities__appropriation_date__gte=from_date,
304316
)
305-
306-
main_activities_appropriated_before_from_date_q = Q(
307-
activities__activity_type=MAIN_ACTIVITY,
308-
activities__appropriation_date__lte=from_date,
317+
# Use subqueries as multiple annotations will yield wrong results:
318+
# https://docs.djangoproject.com/en/2.2/topics/db/aggregation/#combining-multiple-aggregations
319+
# https://code.djangoproject.com/ticket/10060
320+
main_acts = (
321+
Activity.objects.filter(appropriation=OuterRef("id"))
322+
.order_by()
323+
.filter(
324+
activity_type=MAIN_ACTIVITY,
325+
status=STATUS_GRANTED,
326+
)
327+
.values("appropriation")
328+
.annotate(c=Count("appropriation"))
329+
.values("c")
330+
)
331+
main_acts_before_from_date = (
332+
Activity.objects.filter(appropriation=OuterRef("id"))
333+
.order_by()
334+
.filter(
335+
activity_type=MAIN_ACTIVITY,
336+
status=STATUS_GRANTED,
337+
appropriation_date__lt=from_date,
338+
)
339+
.values("appropriation")
340+
.annotate(c=Count("appropriation"))
341+
.values("c")
342+
)
343+
main_acts_after_from_date = (
344+
Activity.objects.filter(appropriation=OuterRef("id"))
345+
.order_by()
346+
.filter(
347+
activity_type=MAIN_ACTIVITY,
348+
status=STATUS_GRANTED,
349+
appropriation_date__gte=from_date,
350+
)
351+
.values("appropriation")
352+
.annotate(c=Count("appropriation"))
353+
.values("c")
309354
)
310355

311-
queryset = queryset.annotate(
312-
main_acts_count=Count("activities", filter=main_activities_q),
313-
main_acts_appropriated_after_from_date_count=Count(
314-
"activities",
315-
filter=main_activities_appropriated_after_from_date_q,
316-
),
317-
main_acts_appropriated_before_from_date_count=Count(
318-
"activities",
319-
filter=main_activities_appropriated_before_from_date_q,
320-
),
321-
dst_report_type=Case(
322-
When(
323-
case__in=changed_cases,
324-
then=Value(report_types["CHANGED"]),
356+
queryset = (
357+
# annotate fields we need to determine dst_report_type.
358+
queryset.annotate(
359+
main_acts_count=Subquery(
360+
main_acts, output_field=IntegerField()
325361
),
326-
When(
327-
main_acts_count=F(
328-
"main_acts_appropriated_before_from_date_count"
329-
),
330-
then=Value(""),
362+
main_acts_after_from_date_count=Subquery(
363+
main_acts_after_from_date, output_field=IntegerField()
331364
),
332-
When(
333-
main_acts_count=F(
334-
"main_acts_appropriated_after_from_date_count"
335-
),
336-
then=Value(report_types["NEW"]),
365+
main_acts_before_from_date_count=Subquery(
366+
main_acts_before_from_date, output_field=IntegerField()
337367
),
338-
When(
339-
main_acts_count__gt=F(
340-
"main_acts_appropriated_after_from_date_count"
368+
dst_report_type=Case(
369+
# If an appropriation has a case with a changed
370+
# acting municipality we mark it "changed".
371+
When(
372+
case__in=changed_cases,
373+
then=Value(report_types["CHANGED"]),
374+
),
375+
# If an appropriation has main activities appropriated
376+
# after the from date, and the total number of
377+
# main activities is higher (and thus has
378+
# appropriated main activities before the from_date)
379+
# we consider it "changed".
380+
When(
381+
Q(main_acts_after_from_date_count__gt=0)
382+
& Q(
383+
main_acts_count__gt=F(
384+
"main_acts_after_from_date_count"
385+
)
386+
),
387+
then=Value(report_types["CHANGED"]),
341388
),
342-
then=Value(report_types["CHANGED"]),
389+
# If appropriation has a total number of main
390+
# activities equal to the number of main activities
391+
# appropriated after the from_date we consider it "new"
392+
When(
393+
Q(
394+
main_acts_count=F(
395+
"main_acts_after_from_date_count"
396+
)
397+
),
398+
then=Value(report_types["NEW"]),
399+
),
400+
# Lastly if an appropriation only has main activities
401+
# appropriated in the past we do not include them.
402+
When(
403+
main_acts_count=F(
404+
"main_acts_before_from_date_count"
405+
),
406+
then=Value(""),
407+
),
408+
default=Value(""),
409+
output_field=CharField(),
343410
),
344-
default=Value(""),
345-
output_field=CharField(),
346-
),
347-
).exclude(dst_report_type="")
348-
else:
349-
queryset = queryset.annotate(
350-
dst_report_type=Value(
351-
report_types["NEW"], output_field=CharField()
352411
)
412+
.exclude(dst_report_type="")
413+
.distinct()
353414
)
354415

355-
return queryset.distinct()
416+
return queryset
356417

357418
def get_duplicate_sbsys_id_appropriations_for_dst(self):
358419
"""
@@ -392,7 +453,7 @@ def get_duplicate_sbsys_id_appropriations_for_dst(self):
392453
)
393454
.exclude(sbsys_common=None)
394455
.values("sbsys_common")
395-
.annotate(ids=ArrayAgg("id"))
456+
.annotate(ids=ArrayAgg("id", distinct=True))
396457
.annotate(id_count=Func("ids", Value(1), function="array_length"))
397458
.filter(id_count__gt=1)
398459
)

0 commit comments

Comments
 (0)