From 8210b432a937839aa5f4919406ac600746f4250d Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Wed, 6 Apr 2022 13:41:49 +0100 Subject: [PATCH 01/13] feat: initial support for participant groups --- .../macros/participant_list_filter.html | 4 + apollo/models.py | 7 +- apollo/participants/filters.py | 46 ++++++++- apollo/participants/models.py | 72 +++++++++++++- apollo/participants/serializers.py | 94 ++++++++++++++++++- apollo/participants/services.py | 12 ++- apollo/participants/tasks.py | 48 ++++++++++ apollo/services.py | 5 +- .../3679afe4da46_add_participant_groups.py | 67 +++++++++++++ 9 files changed, 343 insertions(+), 12 deletions(-) create mode 100644 migrations/versions/3679afe4da46_add_participant_groups.py diff --git a/apollo/frontend/templates/frontend/macros/participant_list_filter.html b/apollo/frontend/templates/frontend/macros/participant_list_filter.html index 6bf68644e..25b5c838e 100644 --- a/apollo/frontend/templates/frontend/macros/participant_list_filter.html +++ b/apollo/frontend/templates/frontend/macros/participant_list_filter.html @@ -33,6 +33,10 @@ {{ form.partner(class_='form-control custom-select') }} +
+ + {{ form.group(class_='form-control select2', size=1, placeholder=_('All Groups')) }} +
diff --git a/apollo/models.py b/apollo/models.py index cf6375a38..207040ca9 100644 --- a/apollo/models.py +++ b/apollo/models.py @@ -7,9 +7,10 @@ LocationTypePath, LocationGroup, locations_groups) from apollo.messaging.models import Message # noqa from apollo.participants.models import ( # noqa - ParticipantSet, ParticipantDataField, - Participant, ParticipantPartner, ParticipantRole, PhoneContact, - ContactHistory, Sample, samples_participants) + ParticipantGroup, ParticipantGroupType, ParticipantSet, + ParticipantDataField, Participant, ParticipantPartner, + ParticipantRole, PhoneContact, ContactHistory, Sample, + groups_participants, samples_participants) from apollo.submissions.models import ( # noqa Submission, SubmissionComment, SubmissionImageAttachment, SubmissionVersion) diff --git a/apollo/participants/filters.py b/apollo/participants/filters.py index cb28b0563..18c5b16da 100644 --- a/apollo/participants/filters.py +++ b/apollo/participants/filters.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from cgi import escape +from collections import OrderedDict from flask_babelex import lazy_gettext as _ from sqlalchemy import func, or_, text, true @@ -12,9 +13,11 @@ from apollo.core import CharFilter, ChoiceFilter, FilterSet from apollo.helpers import _make_choices from apollo.locations.models import Location, LocationPath +from apollo.wtforms_ext import ExtendedMultipleSelectField -from .models import Participant, ParticipantRole, ParticipantPartner -from .models import PhoneContact, Sample +from .models import Participant, ParticipantGroup, ParticipantGroupType +from .models import ParticipantPartner, ParticipantRole +from .models import PhoneContact, Sample, groups_participants class ParticipantIDFilter(CharFilter): @@ -94,6 +97,44 @@ def queryset_(self, query, value): return ParticipantRoleFilter +def make_participant_group_filter(participant_set_id): + class ParticipantGroupFilter(ChoiceFilter): + field_class = ExtendedMultipleSelectField + + def __init__(self, *args, **kwargs): + choices = OrderedDict() + for group_type in services.participant_group_types.find( + participant_set_id=participant_set_id + ).order_by( + ParticipantGroupType.name): + for group in services.participant_groups.find( + group_type=group_type + ).order_by(ParticipantGroup.name): + choices.setdefault(group_type.name, []).append( + (group.id, group.name) + ) + + kwargs['choices'] = [(k, choices[k]) for k in choices] + kwargs['coerce'] = int + super(ParticipantGroupFilter, self).__init__(*args, **kwargs) + + def queryset_(self, query, values): + if values: + query2 = query.join(groups_participants).join( + ParticipantGroup) + return query2.filter( + Participant.id == + models.groups_participants.c.participant_id, # noqa + ParticipantGroup.id == + groups_participants.c.group_id, + ParticipantGroup.id.in_(values) + ) + + return query + + return ParticipantGroupFilter + + def make_participant_partner_filter(participant_set_id): class ParticipantPartnerFilter(ChoiceFilter): def __init__(self, *args, **kwargs): @@ -201,6 +242,7 @@ def participant_filterset(participant_set_id, location_set_id=None): 'name': ParticipantNameFilter(), 'phone': ParticipantPhoneFilter(), 'role': make_participant_role_filter(participant_set_id)(), + 'group': make_participant_group_filter(participant_set_id)(), 'partner': make_participant_partner_filter(participant_set_id)() } diff --git a/apollo/participants/models.py b/apollo/participants/models.py index f531a2477..cbfa2b87a 100644 --- a/apollo/participants/models.py +++ b/apollo/participants/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from itertools import chain import re from sqlalchemy import func @@ -51,6 +50,7 @@ def get_import_fields(self): 'phone': _('Phone'), 'partner': _('Partner'), 'location': _('Location code'), + 'group': _('Group'), 'gender': _('Gender'), 'email': _('Email'), 'password': _('Password') @@ -94,6 +94,20 @@ def get_import_fields(self): ), ) +groups_participants = db.Table( + 'participant_group_participants', + db.Column( + 'group_id', db.Integer, + db.ForeignKey('participant_group.id', ondelete='CASCADE'), + nullable=False + ), + db.Column( + 'participant_id', db.Integer, + db.ForeignKey('participant.id', ondelete='CASCADE'), + nullable=False + ), +) + class Sample(BaseModel): __tablename__ = "sample" @@ -152,6 +166,55 @@ def __str__(self): return self.name or '' +class ParticipantGroupType(BaseModel): + __tablename__ = 'participant_group_type' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String) + participant_set_id = db.Column( + db.Integer, + db.ForeignKey('participant_set.id', ondelete='CASCADE'), + nullable=False + ) + + participant_set = db.relationship( + 'ParticipantSet', backref=db.backref( + 'participant_group_types', cascade='all, delete', + ) + ) + + def __str__(self): + return self.name or '' + + +class ParticipantGroup(BaseModel): + __tablename__ = 'participant_group' + + id = db.Column(db.Integer, primary_key=True) + group_type_id = db.Column( + db.Integer, + db.ForeignKey('participant_group_type.id', ondelete='CASCADE'), + nullable=False + ) + participant_set_id = db.Column( + db.Integer, + db.ForeignKey('participant_set.id', ondelete='CASCADE'), + nullable=False + ) + + group_type = db.relationship( + 'ParticipantGroupType', + backref=db.backref('participant_groups', cascade='all, delete'), + ) + participant_set = db.relationship( + 'ParticipantSet', + backref=db.backref('participant_groups', cascade='all, delete'), + ) + + def __str__(self): + return self.name or '' + + class ParticipantPartner(BaseModel): __tablename__ = 'participant_partner' @@ -227,6 +290,10 @@ class Participant(BaseModel): backref="participants", secondary=samples_participants, ) + groups = db.relationship( + 'ParticipantGroup', secondary=groups_participants, + backref='participants', + ) def __str__(self): return self.name or '' @@ -334,7 +401,8 @@ class PhoneContact(BaseModel): onupdate=utils.current_timestamp) verified = db.Column(db.Boolean, default=False) - participant = db.relationship('Participant', back_populates='phone_contacts') + participant = db.relationship( + 'Participant', back_populates='phone_contacts') def touch(self): self.updated = utils.current_timestamp() diff --git a/apollo/participants/serializers.py b/apollo/participants/serializers.py index 8d108f253..206ebd26a 100644 --- a/apollo/participants/serializers.py +++ b/apollo/participants/serializers.py @@ -7,8 +7,9 @@ from apollo.dal.serializers import ArchiveSerializer from apollo.locations.models import Location, LocationSet from apollo.participants.models import ( - Participant, ParticipantDataField, ParticipantPartner, ParticipantRole, - ParticipantSet, PhoneContact) + Participant, ParticipantDataField, ParticipantGroup, + ParticipantGroupType, ParticipantPartner, ParticipantRole, + ParticipantSet, PhoneContact, groups_participants) class ParticipantSerializer(object): @@ -105,6 +106,63 @@ def serialize_one(self, obj): } +class ParticipantGroupTypeSerializer(object): + __model__ = ParticipantGroupType + + def deserialize_one(self, data): + participant_set_id = ParticipantSet.query.filter_by( + uuid=data['participant_set'] + ).with_entities(ParticipantSet.id).scalar() + + kwargs = data.copy() + kwargs.pop('participant_set') + kwargs['participant_set_id'] = participant_set_id + + return self.__model__(**kwargs) + + def serialize_one(self, obj): + if not isinstance(obj, self.__model__): + raise TypeError('Object is not instance of ParticipantGroupType') + + return { + 'uuid': obj.uuid.hex, + 'name': obj.name, + 'participant_set': obj.participant_set.uuid.hex + } + + +class ParticipantGroupSerializer(object): + __model__ = ParticipantGroup + + def deserialize_one(self, data): + participant_set_id = ParticipantSet.query.filter_by( + uuid=data['participant_set'] + ).with_entities(ParticipantSet.id).scalar() + + group_type_id = ParticipantGroupType.query.filter_by( + uuid=data['group_type'] + ).with_entitites(ParticipantGroupType.id).scalar() + + kwargs = data.copy() + kwargs.pop('participant_set') + kwargs.pop('group_type') + kwargs['group_type_id'] = group_type_id + kwargs['participant_set_id'] = participant_set_id + + return self.__model__(**kwargs) + + def serialize_one(self, obj): + if not isinstance(obj, self.__model__): + raise TypeError('Object is not instance of ParticipantGroup') + + return { + 'uuid': obj.uuid.hex, + 'name': obj.name, + 'participant_set': obj.participant_set.uuid.hex, + 'group_type': obj.group_type.uuid.hex + } + + class ParticipantDataFieldSerializer(object): __model__ = ParticipantDataField @@ -195,6 +253,9 @@ def serialize(self, event, zip_file): self.serialize_partners(participant_set.participant_partners, zip_file) self.serialize_roles(participant_set.participant_roles, zip_file) + self.serialize_group_types(participant_set.participant_group_types, + zip_file) + self.serialize_groups(participant_set.participant_groups, zip_file) self.serialize_participants(participant_set.participants, zip_file) self.serialize_participant_phones(participant_set, zip_file) @@ -205,6 +266,35 @@ def serialize_participant_set(self, obj, zip_file): with zip_file.open('participant_set.ndjson', 'w') as f: f.write(json.dumps(data).encode('utf-8')) + def serialize_group_types(self, group_types, zip_file): + serializer = ParticipantGroupTypeSerializer() + + with zip_file.open('group_types.ndjson', 'w') as f: + for group_type in group_types: + data = serializer.serialize_one(group_type) + line = f'{json.dumps(data)}\n' + f.write(line.encode('utf-8')) + + def serialize_groups(self, groups, zip_file): + serializer = ParticipantGroupSerializer() + + with zip_file.open('groups.ndjson', 'w') as f: + for group in groups: + data = serializer.serialize_one(group) + line = f'{json.dumps(data)}\n' + f.write(line.encode('utf-8')) + + def serialize_participant_groups(self, participant_set, zip_file): + query = db.session.query(groups_participants).join( + Participant).join(ParticipantGroup).with_entities( + cast(Participant.uuid, String), + cast(ParticipantGroup.uuid, String)) + + with zip_file.open('participant-groups.ndjson', 'w') as f: + for pair in query: + line = f'{json.dumps(pair)}\n' + f.write(line.encode('utf-8')) + def serialize_extra_fields(self, extra_fields, zip_file): serializer = ParticipantDataFieldSerializer() diff --git a/apollo/participants/services.py b/apollo/participants/services.py index b2b35c75f..cae1ccfeb 100644 --- a/apollo/participants/services.py +++ b/apollo/participants/services.py @@ -8,8 +8,8 @@ from apollo import constants from apollo.dal.service import Service from apollo.participants.models import ( - ParticipantSet, Participant, ParticipantPartner, ParticipantRole, - PhoneContact) + ParticipantSet, Participant, ParticipantGroup, ParticipantGroupType, + ParticipantPartner, ParticipantRole, PhoneContact) number_regex = re.compile('[^0-9]') @@ -99,6 +99,14 @@ def export_list(self, query): output_buffer.close() +class ParticipantGroupService(Service): + __model__ = ParticipantGroup + + +class ParticipantGroupTypeService(Service): + __model__ = ParticipantGroupType + + class ParticipantPartnerService(Service): __model__ = ParticipantPartner diff --git a/apollo/participants/tasks.py b/apollo/participants/tasks.py index 5d410be7e..40c6365d0 100644 --- a/apollo/participants/tasks.py +++ b/apollo/participants/tasks.py @@ -55,6 +55,17 @@ def create_partner(name, participant_set): name=name, participant_set_id=participant_set.id) +def create_group_type(name, participant_set): + return services.participant_group_types.create( + name=name, participant_set_id=participant_set.id) + + +def create_group(name, group_type, participant_set): + return services.participant_groups.create( + name=name, group_type_id=group_type.id, + participant_set_id=participant_set.id) + + def create_role(name, participant_set): return services.participant_roles.create( name=name, participant_set_id=participant_set.id) @@ -86,6 +97,8 @@ def update_participants(dataframe, header_map, participant_set, task): password - the participant's password. phone - a prefix for columns starting with this string that contain numbers + group - a prefix for columns starting with this string that contain + participant group names """ index = dataframe.index @@ -128,6 +141,7 @@ def update_participants(dataframe, header_map, participant_set, task): EMAIL_COL = header_map.get('email') PASSWORD_COL = header_map.get('password') phone_columns = header_map.get('phone', []) + group_columns = header_map.get('group', []) sample_columns = header_map.get('sample', []) full_name_columns = [ header_map.get(col) for col in full_name_column_keys] @@ -371,6 +385,33 @@ def update_participants(dataframe, header_map, participant_set, task): number=mobile_num, participant_id=participant.id, verified=True) + groups = [] + # fix up groups + if group_columns: + for column in group_columns: + if not _is_valid(record[column]): + continue + + group_type = services.participant_group_types.find( + name=column, + participant_set=participant_set + ).first() + + if not group_type: + group_type = create_group_type( + column, participant_set) + + group = services.participant_groups.find( + name=record[column], + group_type=group_type, + participant_set=participant_set).first() + + if not group: + group = create_group( + record[column], group_type, participant_set) + + groups.append(group) + if sample_columns: for column in sample_columns: if not _is_valid(record[column]): @@ -408,6 +449,13 @@ def update_participants(dataframe, header_map, participant_set, task): services.participants.find(id=participant.id).update( {'extra_data': extra_data}, synchronize_session=False) + if groups: + if participant.groups: + participant.groups.extend(groups) + else: + participant.groups = groups + participant.save() + task.update_task_info( total_records=total_records, error_records=error_records, diff --git a/apollo/services.py b/apollo/services.py index d0d74de34..f53d7fd73 100644 --- a/apollo/services.py +++ b/apollo/services.py @@ -5,8 +5,9 @@ LocationService, LocationSetService, LocationTypeService) from apollo.messaging.services import MessageService from apollo.participants.services import ( + ParticipantService, ParticipantGroupService, ParticipantGroupTypeService, ParticipantSetService, ParticipantPartnerService, ParticipantRoleService, - ParticipantService, PhoneContactService) + PhoneContactService) from apollo.submissions.services import ( SubmissionService, SubmissionCommentService, SubmissionVersionService) from apollo.users.services import UserService, UserUploadService @@ -21,6 +22,8 @@ participant_partners = ParticipantPartnerService() participant_roles = ParticipantRoleService() participant_sets = ParticipantSetService() +participant_groups = ParticipantGroupService() +participant_group_types = ParticipantGroupTypeService() phone_contacts = PhoneContactService() submissions = SubmissionService() submission_comments = SubmissionCommentService() diff --git a/migrations/versions/3679afe4da46_add_participant_groups.py b/migrations/versions/3679afe4da46_add_participant_groups.py new file mode 100644 index 000000000..9bfb54779 --- /dev/null +++ b/migrations/versions/3679afe4da46_add_participant_groups.py @@ -0,0 +1,67 @@ +"""add participant groups + +Revision ID: 3679afe4da46 +Revises: c4166678fb79 +Create Date: 2022-04-06 13:17:44.946820 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "3679afe4da46" +down_revision = "c4166678fb79" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "participant_group_type", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(), nullable=True), + sa.Column("participant_set_id", sa.Integer(), nullable=False), + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.ForeignKeyConstraint( + ["participant_set_id"], ["participant_set.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "participant_group", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("group_type_id", sa.Integer(), nullable=False), + sa.Column("participant_set_id", sa.Integer(), nullable=False), + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.ForeignKeyConstraint( + ["group_type_id"], + ["participant_group_type.id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["participant_set_id"], ["participant_set.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "participant_group_participants", + sa.Column("group_id", sa.Integer(), nullable=False), + sa.Column("participant_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["group_id"], ["participant_group.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["participant_id"], ["participant.id"], ondelete="CASCADE" + ), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("participant_group_participants") + op.drop_table("participant_group") + op.drop_table("participant_group_type") + # ### end Alembic commands ### From a1500dc1f8381ac116eb4a222b3787e57aa05614 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Fri, 8 Apr 2022 08:55:34 +0100 Subject: [PATCH 02/13] fix: add the `text/plain` mimetype for CSV imports --- apollo/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apollo/helpers.py b/apollo/helpers.py index 333835862..3c129a28d 100644 --- a/apollo/helpers.py +++ b/apollo/helpers.py @@ -7,6 +7,7 @@ import pkgutil CSV_MIMETYPES = [ + "text/plain", "text/csv", "application/csv", "text/x-csv", From 9ba59b0b96d18188ba49a69f1db7c1be675d4611 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Fri, 8 Apr 2022 09:02:37 +0100 Subject: [PATCH 03/13] fix: add missing `name` column --- apollo/participants/models.py | 1 + migrations/versions/3679afe4da46_add_participant_groups.py | 1 + 2 files changed, 2 insertions(+) diff --git a/apollo/participants/models.py b/apollo/participants/models.py index cbfa2b87a..5b1b1d40b 100644 --- a/apollo/participants/models.py +++ b/apollo/participants/models.py @@ -191,6 +191,7 @@ class ParticipantGroup(BaseModel): __tablename__ = 'participant_group' id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String, nullable=False) group_type_id = db.Column( db.Integer, db.ForeignKey('participant_group_type.id', ondelete='CASCADE'), diff --git a/migrations/versions/3679afe4da46_add_participant_groups.py b/migrations/versions/3679afe4da46_add_participant_groups.py index 9bfb54779..720b429c8 100644 --- a/migrations/versions/3679afe4da46_add_participant_groups.py +++ b/migrations/versions/3679afe4da46_add_participant_groups.py @@ -32,6 +32,7 @@ def upgrade(): op.create_table( "participant_group", sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.VARCHAR(), nullable=False), sa.Column("group_type_id", sa.Integer(), nullable=False), sa.Column("participant_set_id", sa.Integer(), nullable=False), sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), From f203bcdaa33fc1e523d694583ce6fd8dfe599301 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Sat, 2 Jul 2022 13:47:16 +0100 Subject: [PATCH 04/13] feat: filter submission lists by participant group --- .../macros/submission_list_filter.html | 4 ++ .../templates/frontend/submission_list.html | 5 ++ apollo/submissions/filters.py | 46 ++++++++++++++++++- 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/apollo/frontend/templates/frontend/macros/submission_list_filter.html b/apollo/frontend/templates/frontend/macros/submission_list_filter.html index fd6d8e23d..1c054397f 100644 --- a/apollo/frontend/templates/frontend/macros/submission_list_filter.html +++ b/apollo/frontend/templates/frontend/macros/submission_list_filter.html @@ -70,6 +70,10 @@ {{ filter_form.participant_role(class_='form-control custom-select') }}
+
+ + {{ filter_form.participant_group(class_='form-control select2', size=1, placeholder=_('All Participant Groups')) }} +
{%- if form.show_moment %}
diff --git a/apollo/frontend/templates/frontend/submission_list.html b/apollo/frontend/templates/frontend/submission_list.html index 05155fb25..3226a1c88 100644 --- a/apollo/frontend/templates/frontend/submission_list.html +++ b/apollo/frontend/templates/frontend/submission_list.html @@ -24,6 +24,11 @@ var map = undefined; LocationOptions.placeholder = { id: '-1', text: '{{ _("Location") }}'}; + $('#{{ filter_form.participant_group.id }}').select2({ + theme: 'bootstrap4', + placeholder: "{{ _('All Participant Groups') }}" + }); + $.fn.datetimepicker.Constructor.Default = $.extend({}, $.fn.datetimepicker.Constructor.Default, { format: 'DD-MM-YYYY', widgetPositioning: { diff --git a/apollo/submissions/filters.py b/apollo/submissions/filters.py index 81896e82f..155212d2a 100644 --- a/apollo/submissions/filters.py +++ b/apollo/submissions/filters.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from collections import OrderedDict from itertools import chain from operator import itemgetter @@ -13,7 +14,7 @@ from wtforms.widgets import html_params, HTMLString from wtforms_alchemy.fields import QuerySelectField -from apollo import models +from apollo import models, services from apollo.core import BooleanFilter, CharFilter, ChoiceFilter, FilterSet from apollo.helpers import _make_choices from apollo.settings import TIMEZONE @@ -21,6 +22,7 @@ from apollo.submissions.qa.query_builder import ( build_expression, generate_qa_query ) +from apollo.wtforms_ext import ExtendedMultipleSelectField APP_TZ = gettz(TIMEZONE) @@ -170,6 +172,46 @@ def queryset_(self, query, value, **kwargs): return SubmissionLocationGroupFilter +def make_participant_group_filter(participant_set_id): + class ParticipantGroupFilter(ChoiceFilter): + field_class = ExtendedMultipleSelectField + + def __init__(self, *args, **kwargs): + self.participant_set_id = participant_set_id + + choices = OrderedDict() + for group_type in services.participant_group_types.find( + participant_set_id=participant_set_id + ).order_by(models.ParticipantGroupType.name): + for group in services.participant_groups.find( + group_type=group_type + ).order_by(models.ParticipantGroup.name): + choices.setdefault(group_type.name, []).append( + (group.id, group.name) + ) + + kwargs['choices'] = [(k, choices[k]) for k in choices] + kwargs['coerce'] = int + super(ParticipantGroupFilter, self).__init__(*args, **kwargs) + + def queryset_(self, queryset, values): + if values: + participant_ids = models.Participant.query.join( + models.Participant.groups + ).filter( + models.Participant.participant_set_id == self.participant_set_id, # noqa + models.ParticipantGroup.id.in_(values) + ).with_entities(models.Participant.id) + + return queryset.filter( + models.Submission.participant_id.in_(participant_ids) + ) + + return queryset + + return ParticipantGroupFilter + + def make_base_submission_filter(event, filter_on_locations=False): class BaseSubmissionFilterSet(FilterSet): sample = make_submission_sample_filter( @@ -748,6 +790,8 @@ def make_submission_list_filter(event, form, filter_on_locations=False): attributes['fsn'] = FormSerialNumberFilter() attributes['participant_role'] = make_participant_role_filter( event.participant_set_id)() + attributes['participant_group'] = make_participant_group_filter( + event.participant_set_id)() return type( 'SubmissionFilterSet', From 74de4e788e02e72dddf4bbd7b166ac90fe673840 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Sat, 2 Jul 2022 14:14:20 +0100 Subject: [PATCH 05/13] feat: filter dashboard by participant group --- .../templates/frontend/dashboard.html | 7 ++++- .../frontend/macros/dashboard_filter.html | 31 ++++++++++++++++++- apollo/submissions/filters.py | 2 ++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/apollo/frontend/templates/frontend/dashboard.html b/apollo/frontend/templates/frontend/dashboard.html index ce2849afb..6ddf02f3f 100644 --- a/apollo/frontend/templates/frontend/dashboard.html +++ b/apollo/frontend/templates/frontend/dashboard.html @@ -37,7 +37,7 @@ {% block content %}
- {{ render_filter_form(filter_form, location)}} + {{ render_filter_form(form, filter_form, location)}}
{%- if not daily_stratified_progress %} @@ -150,6 +150,11 @@
{% if not location and not request.args.location %} moment.lang('{{ g.locale }}'); + $('#{{ filter_form.participant_group.id }}').select2({ + theme: 'bootstrap4', + placeholder: "{{ _('All Participant Groups') }}" + }); + $('.timestamp-date').each(function (index) { var timestamp = moment.unix(Number($(this).data('timestamp'))); this.innerText = timestamp.format('MMM D'); diff --git a/apollo/frontend/templates/frontend/macros/dashboard_filter.html b/apollo/frontend/templates/frontend/macros/dashboard_filter.html index ad12af51e..19d6dc742 100644 --- a/apollo/frontend/templates/frontend/macros/dashboard_filter.html +++ b/apollo/frontend/templates/frontend/macros/dashboard_filter.html @@ -1,7 +1,35 @@ -{% macro render_filter_form(filter_form, location) %} +{% macro render_filter_form(dashboard_form, filter_form, location) %}
+ {% if dashboard_form.untrack_data_conflicts %} +
+
+ + {{ filter_form.sample(class_='form-control custom-select') }} +
+
+ + {{ filter_form.location_group(class_='form-control custom-select') }} +
+
+ + {{ filter_form.participant_group(class_='form-control select2', size=1, placeholder=_('All Participant Groups')) }} +
+
+
+
+ + {{ filter_form.location(class_='form-control select2 select2-locations') }} +
+
+
+ + {{ _('Clear') }} +
+
+
+ {% else %}
@@ -22,6 +50,7 @@
+ {% endif %}
diff --git a/apollo/submissions/filters.py b/apollo/submissions/filters.py index 155212d2a..d905bd91f 100644 --- a/apollo/submissions/filters.py +++ b/apollo/submissions/filters.py @@ -718,6 +718,8 @@ def make_dashboard_filter(event, filter_on_locations=False): event.participant_set_id, filter_on_locations=filter_on_locations)() attributes['location_group'] = make_submission_location_group_filter( event.location_set_id)() + attributes['participant_group'] = make_participant_group_filter( + event.participant_set_id)() return type( 'SubmissionFilterSet', From 40b7171a7412ee35918cdf8018b8e33c02cf3f25 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Sun, 3 Jul 2022 19:13:08 +0100 Subject: [PATCH 06/13] feat: add extra check --- apollo/frontend/templates/frontend/macros/dashboard_filter.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apollo/frontend/templates/frontend/macros/dashboard_filter.html b/apollo/frontend/templates/frontend/macros/dashboard_filter.html index 19d6dc742..ea56eaeb1 100644 --- a/apollo/frontend/templates/frontend/macros/dashboard_filter.html +++ b/apollo/frontend/templates/frontend/macros/dashboard_filter.html @@ -2,7 +2,7 @@
- {% if dashboard_form.untrack_data_conflicts %} + {% if dashboard_form.untrack_data_conflicts and dashboard_form.form_type == 'CHECKLIST' %}
From f40251496750d59666920e4a4461e0b9531e7956 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Sun, 3 Jul 2022 19:28:28 +0100 Subject: [PATCH 07/13] fix: correct conditional expression --- apollo/frontend/templates/frontend/macros/dashboard_filter.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apollo/frontend/templates/frontend/macros/dashboard_filter.html b/apollo/frontend/templates/frontend/macros/dashboard_filter.html index ea56eaeb1..d2c2b6d62 100644 --- a/apollo/frontend/templates/frontend/macros/dashboard_filter.html +++ b/apollo/frontend/templates/frontend/macros/dashboard_filter.html @@ -2,7 +2,7 @@
- {% if dashboard_form.untrack_data_conflicts and dashboard_form.form_type == 'CHECKLIST' %} + {% if dashboard_form.untrack_data_conflicts or dashboard_form.form_type != 'CHECKLIST' %}
From 586ef6f2e545f57bdffbdd043e28a4453d3cd962 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Sun, 3 Jul 2022 19:43:22 +0100 Subject: [PATCH 08/13] feat: filter QA list using participant group --- .../frontend/macros/quality_assurance_list_filter.html | 4 ++++ .../frontend/templates/frontend/quality_assurance_list.html | 5 +++++ apollo/submissions/filters.py | 2 ++ 3 files changed, 11 insertions(+) diff --git a/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html b/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html index fbe2c2e91..6ed4d8cde 100644 --- a/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html +++ b/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html @@ -45,6 +45,10 @@ {{ filter_form.participant_role(class_='form-control custom-select') }}
+
+ + {{ filter_form.participant_group(class_='form-control select2', size=1, placeholder=_('All Participant Groups')) }} +
{%- if form.show_moment %}
diff --git a/apollo/frontend/templates/frontend/quality_assurance_list.html b/apollo/frontend/templates/frontend/quality_assurance_list.html index dedc822f5..51e4c323e 100644 --- a/apollo/frontend/templates/frontend/quality_assurance_list.html +++ b/apollo/frontend/templates/frontend/quality_assurance_list.html @@ -41,6 +41,11 @@ } }); + $('#{{ filter_form.participant_group.id }}').select2({ + theme: 'bootstrap4', + placeholder: "{{ _('All Participant Groups') }}" + }); + $('#filter_reset').on('click', function() { var $form = $(this).parents('form').first(); $form.find(':input').not('button').each(function() { $(this).val(''); }) diff --git a/apollo/submissions/filters.py b/apollo/submissions/filters.py index d905bd91f..91b9d3c17 100644 --- a/apollo/submissions/filters.py +++ b/apollo/submissions/filters.py @@ -852,6 +852,8 @@ class QualityAssuranceConditionsForm(Form): attributes['fsn'] = FormSerialNumberFilter() attributes['participant_role'] = make_participant_role_filter( event.participant_set_id)() + attributes['participant_group'] = make_participant_group_filter( + event.participant_set_id)() return type( 'QualityAssuranceFilterSet', From 7b9bea2f337a8a4afdeefa12cd5e09524be70ec7 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Wed, 6 Jul 2022 02:11:51 +0100 Subject: [PATCH 09/13] chore: change label and styling --- apollo/frontend/templates/frontend/dashboard.html | 2 +- apollo/frontend/templates/frontend/macros/dashboard_filter.html | 2 +- .../frontend/macros/quality_assurance_list_filter.html | 2 +- .../templates/frontend/macros/submission_list_filter.html | 2 +- apollo/frontend/templates/frontend/quality_assurance_list.html | 2 +- apollo/frontend/templates/frontend/submission_list.html | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apollo/frontend/templates/frontend/dashboard.html b/apollo/frontend/templates/frontend/dashboard.html index 6ddf02f3f..2ad775009 100644 --- a/apollo/frontend/templates/frontend/dashboard.html +++ b/apollo/frontend/templates/frontend/dashboard.html @@ -152,7 +152,7 @@
{% if not location and not request.args.location %}
- {{ filter_form.participant_group(class_='form-control select2', size=1, placeholder=_('All Participant Groups')) }} + {{ filter_form.participant_group(class_='form-control custom-select', size=1, placeholder=_('Participant Groups')) }}
diff --git a/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html b/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html index 6ed4d8cde..3ea90d81a 100644 --- a/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html +++ b/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html @@ -47,7 +47,7 @@
- {{ filter_form.participant_group(class_='form-control select2', size=1, placeholder=_('All Participant Groups')) }} + {{ filter_form.participant_group(class_='form-control custom-select', size=1, placeholder=_('Participant Groups')) }}
{%- if form.show_moment %}
diff --git a/apollo/frontend/templates/frontend/macros/submission_list_filter.html b/apollo/frontend/templates/frontend/macros/submission_list_filter.html index 1c054397f..30926b7f0 100644 --- a/apollo/frontend/templates/frontend/macros/submission_list_filter.html +++ b/apollo/frontend/templates/frontend/macros/submission_list_filter.html @@ -72,7 +72,7 @@
- {{ filter_form.participant_group(class_='form-control select2', size=1, placeholder=_('All Participant Groups')) }} + {{ filter_form.participant_group(class_='form-control custom-select', size=1, placeholder=_('Participant Groups')) }}
{%- if form.show_moment %}
diff --git a/apollo/frontend/templates/frontend/quality_assurance_list.html b/apollo/frontend/templates/frontend/quality_assurance_list.html index 51e4c323e..f3fad05e3 100644 --- a/apollo/frontend/templates/frontend/quality_assurance_list.html +++ b/apollo/frontend/templates/frontend/quality_assurance_list.html @@ -43,7 +43,7 @@ $('#{{ filter_form.participant_group.id }}').select2({ theme: 'bootstrap4', - placeholder: "{{ _('All Participant Groups') }}" + placeholder: "{{ _('Participant Groups') }}" }); $('#filter_reset').on('click', function() { diff --git a/apollo/frontend/templates/frontend/submission_list.html b/apollo/frontend/templates/frontend/submission_list.html index 3226a1c88..2a6237c7a 100644 --- a/apollo/frontend/templates/frontend/submission_list.html +++ b/apollo/frontend/templates/frontend/submission_list.html @@ -26,7 +26,7 @@ $('#{{ filter_form.participant_group.id }}').select2({ theme: 'bootstrap4', - placeholder: "{{ _('All Participant Groups') }}" + placeholder: "{{ _('Participant Groups') }}" }); $.fn.datetimepicker.Constructor.Default = $.extend({}, $.fn.datetimepicker.Constructor.Default, { From d2ea45fadd1b96237457d0b42d034f071f9b36a5 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Tue, 12 Jul 2022 05:48:16 +0100 Subject: [PATCH 10/13] fix: update definition from source --- apollo/wtforms_ext.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apollo/wtforms_ext.py b/apollo/wtforms_ext.py index e41a67b5a..60c87f016 100644 --- a/apollo/wtforms_ext.py +++ b/apollo/wtforms_ext.py @@ -25,10 +25,8 @@ def __call__(self, field, **kwargs): group_items = item2 html.append('' % html_params(label=group_label)) for inner_val, inner_label in group_items: - if field.data: - html.append(self.render_option(inner_val, inner_label, inner_val in field.data)) - else: - html.append(self.render_option(inner_val, inner_label, inner_val == field.data)) + html.append( + self.render_option(inner_val, inner_label, inner_val == field.data)) html.append('') else: val = item1 From f77b9673ec07ccf7c8438803706ac6350ad45489 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Tue, 12 Jul 2022 05:50:09 +0100 Subject: [PATCH 11/13] chore: do not initialize select2 on group widgets --- apollo/frontend/templates/frontend/dashboard.html | 5 ----- apollo/frontend/templates/frontend/participant_list.html | 4 ---- .../frontend/templates/frontend/quality_assurance_list.html | 5 ----- apollo/frontend/templates/frontend/submission_list.html | 5 ----- 4 files changed, 19 deletions(-) diff --git a/apollo/frontend/templates/frontend/dashboard.html b/apollo/frontend/templates/frontend/dashboard.html index 2ad775009..90dc1acb5 100644 --- a/apollo/frontend/templates/frontend/dashboard.html +++ b/apollo/frontend/templates/frontend/dashboard.html @@ -150,11 +150,6 @@
{% if not location and not request.args.location %} moment.lang('{{ g.locale }}'); - $('#{{ filter_form.participant_group.id }}').select2({ - theme: 'bootstrap4', - placeholder: "{{ _('Participant Groups') }}" - }); - $('.timestamp-date').each(function (index) { var timestamp = moment.unix(Number($(this).data('timestamp'))); this.innerText = timestamp.format('MMM D'); diff --git a/apollo/frontend/templates/frontend/participant_list.html b/apollo/frontend/templates/frontend/participant_list.html index 0d7beb31b..5ee3868ce 100644 --- a/apollo/frontend/templates/frontend/participant_list.html +++ b/apollo/frontend/templates/frontend/participant_list.html @@ -356,10 +356,6 @@

{{ _('Finalize') }}

LocationOptions.placeholder = { id: '-1', text: '{{ _("Location") }}'}; $('select.select2-locations').select2(LocationOptions); - $('#group.select2').select2({ - theme: 'bootstrap4', - placeholder: "{{ _('All Groups') }}" - }); }); diff --git a/apollo/frontend/templates/frontend/quality_assurance_list.html b/apollo/frontend/templates/frontend/quality_assurance_list.html index f3fad05e3..dedc822f5 100644 --- a/apollo/frontend/templates/frontend/quality_assurance_list.html +++ b/apollo/frontend/templates/frontend/quality_assurance_list.html @@ -41,11 +41,6 @@ } }); - $('#{{ filter_form.participant_group.id }}').select2({ - theme: 'bootstrap4', - placeholder: "{{ _('Participant Groups') }}" - }); - $('#filter_reset').on('click', function() { var $form = $(this).parents('form').first(); $form.find(':input').not('button').each(function() { $(this).val(''); }) diff --git a/apollo/frontend/templates/frontend/submission_list.html b/apollo/frontend/templates/frontend/submission_list.html index 2a6237c7a..05155fb25 100644 --- a/apollo/frontend/templates/frontend/submission_list.html +++ b/apollo/frontend/templates/frontend/submission_list.html @@ -24,11 +24,6 @@ var map = undefined; LocationOptions.placeholder = { id: '-1', text: '{{ _("Location") }}'}; - $('#{{ filter_form.participant_group.id }}').select2({ - theme: 'bootstrap4', - placeholder: "{{ _('Participant Groups') }}" - }); - $.fn.datetimepicker.Constructor.Default = $.extend({}, $.fn.datetimepicker.Constructor.Default, { format: 'DD-MM-YYYY', widgetPositioning: { From 223bfa458fe363ab396f9e356a5081c9c709ace9 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Tue, 12 Jul 2022 05:54:31 +0100 Subject: [PATCH 12/13] chore: use extended select field and add labels this commit changes the empty label for location group filter fields to a localized translation of "Location Group" it also adds an empty label for participant group filter fields and makes it single-choice, not multiple. --- apollo/participants/filters.py | 11 ++++++----- apollo/submissions/filters.py | 15 +++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/apollo/participants/filters.py b/apollo/participants/filters.py index 18c5b16da..2f139f29d 100644 --- a/apollo/participants/filters.py +++ b/apollo/participants/filters.py @@ -13,7 +13,7 @@ from apollo.core import CharFilter, ChoiceFilter, FilterSet from apollo.helpers import _make_choices from apollo.locations.models import Location, LocationPath -from apollo.wtforms_ext import ExtendedMultipleSelectField +from apollo.wtforms_ext import ExtendedSelectField from .models import Participant, ParticipantGroup, ParticipantGroupType from .models import ParticipantPartner, ParticipantRole @@ -99,10 +99,11 @@ def queryset_(self, query, value): def make_participant_group_filter(participant_set_id): class ParticipantGroupFilter(ChoiceFilter): - field_class = ExtendedMultipleSelectField + field_class = ExtendedSelectField def __init__(self, *args, **kwargs): choices = OrderedDict() + choices[''] = _('Group') for group_type in services.participant_group_types.find( participant_set_id=participant_set_id ).order_by( @@ -118,8 +119,8 @@ def __init__(self, *args, **kwargs): kwargs['coerce'] = int super(ParticipantGroupFilter, self).__init__(*args, **kwargs) - def queryset_(self, query, values): - if values: + def queryset_(self, query, value): + if value: query2 = query.join(groups_participants).join( ParticipantGroup) return query2.filter( @@ -127,7 +128,7 @@ def queryset_(self, query, values): models.groups_participants.c.participant_id, # noqa ParticipantGroup.id == groups_participants.c.group_id, - ParticipantGroup.id.in_(values) + ParticipantGroup.id == value, ) return query diff --git a/apollo/submissions/filters.py b/apollo/submissions/filters.py index 91b9d3c17..7af97c629 100644 --- a/apollo/submissions/filters.py +++ b/apollo/submissions/filters.py @@ -22,7 +22,7 @@ from apollo.submissions.qa.query_builder import ( build_expression, generate_qa_query ) -from apollo.wtforms_ext import ExtendedMultipleSelectField +from apollo.wtforms_ext import ExtendedSelectField APP_TZ = gettz(TIMEZONE) @@ -152,7 +152,8 @@ def __init__(self, *args, **kwargs): ).all() self.location_set_id = location_set_id - kwargs['choices'] = _make_choices(group_choices, _('Group')) + kwargs['choices'] = _make_choices( + group_choices, _('Location Group')) super().__init__(*args, **kwargs) def queryset_(self, query, value, **kwargs): @@ -174,12 +175,13 @@ def queryset_(self, query, value, **kwargs): def make_participant_group_filter(participant_set_id): class ParticipantGroupFilter(ChoiceFilter): - field_class = ExtendedMultipleSelectField + field_class = ExtendedSelectField def __init__(self, *args, **kwargs): self.participant_set_id = participant_set_id choices = OrderedDict() + choices[''] = _('Participant Group') for group_type in services.participant_group_types.find( participant_set_id=participant_set_id ).order_by(models.ParticipantGroupType.name): @@ -191,16 +193,17 @@ def __init__(self, *args, **kwargs): ) kwargs['choices'] = [(k, choices[k]) for k in choices] + print(kwargs['choices']) kwargs['coerce'] = int super(ParticipantGroupFilter, self).__init__(*args, **kwargs) - def queryset_(self, queryset, values): - if values: + def queryset_(self, queryset, value): + if value: participant_ids = models.Participant.query.join( models.Participant.groups ).filter( models.Participant.participant_set_id == self.participant_set_id, # noqa - models.ParticipantGroup.id.in_(values) + models.ParticipantGroup.id == value ).with_entities(models.Participant.id) return queryset.filter( From 800432337efd272721514040360013488e668155 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Tue, 12 Jul 2022 05:57:43 +0100 Subject: [PATCH 13/13] chore: change rendering of participant groups --- apollo/frontend/templates/frontend/macros/dashboard_filter.html | 2 +- .../templates/frontend/macros/participant_list_filter.html | 2 +- .../frontend/macros/quality_assurance_list_filter.html | 2 +- .../templates/frontend/macros/submission_list_filter.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apollo/frontend/templates/frontend/macros/dashboard_filter.html b/apollo/frontend/templates/frontend/macros/dashboard_filter.html index 6708e7ed0..145ef90b4 100644 --- a/apollo/frontend/templates/frontend/macros/dashboard_filter.html +++ b/apollo/frontend/templates/frontend/macros/dashboard_filter.html @@ -14,7 +14,7 @@
- {{ filter_form.participant_group(class_='form-control custom-select', size=1, placeholder=_('Participant Groups')) }} + {{ filter_form.participant_group(class_='form-control custom-select') }}
diff --git a/apollo/frontend/templates/frontend/macros/participant_list_filter.html b/apollo/frontend/templates/frontend/macros/participant_list_filter.html index 25b5c838e..4d1aaed41 100644 --- a/apollo/frontend/templates/frontend/macros/participant_list_filter.html +++ b/apollo/frontend/templates/frontend/macros/participant_list_filter.html @@ -35,7 +35,7 @@
- {{ form.group(class_='form-control select2', size=1, placeholder=_('All Groups')) }} + {{ form.group(class_='form-control custom-select') }}
diff --git a/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html b/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html index 3ea90d81a..545fbf0c6 100644 --- a/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html +++ b/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html @@ -47,7 +47,7 @@
- {{ filter_form.participant_group(class_='form-control custom-select', size=1, placeholder=_('Participant Groups')) }} + {{ filter_form.participant_group(class_='form-control custom-select') }}
{%- if form.show_moment %}
diff --git a/apollo/frontend/templates/frontend/macros/submission_list_filter.html b/apollo/frontend/templates/frontend/macros/submission_list_filter.html index 30926b7f0..9eaa0e573 100644 --- a/apollo/frontend/templates/frontend/macros/submission_list_filter.html +++ b/apollo/frontend/templates/frontend/macros/submission_list_filter.html @@ -72,7 +72,7 @@
- {{ filter_form.participant_group(class_='form-control custom-select', size=1, placeholder=_('Participant Groups')) }} + {{ filter_form.participant_group(class_='form-control custom-select') }}
{%- if form.show_moment %}