import_votes.py 4.16 KB
Newer Older
Nicolas Joyard's avatar
Nicolas Joyard committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# coding: utf-8

import ijson
import logging
import sys

import django
from django.apps import apps
from django.utils.text import slugify

from representatives_votes.models import Proposal, Representative, Vote

logger = logging.getLogger(__name__)


class VotesImporter:
17
18
    deputes_slug = None
    deputes_rid = None
Nicolas Joyard's avatar
Nicolas Joyard committed
19
20
21
22
23
24
25
26
27
    scrutins = None
    touched = []

    positions = dict(
        pour="for",
        contre="against",
        abstention="abstain"
    )

28
29
30
    def get_depute_by_name(self, prenom, nom):
        if self.deputes_slug is None:
            self.deputes_slug = {
Nicolas Joyard's avatar
Nicolas Joyard committed
31
32
33
34
35
                slugify(r[0]): r[1] for r in
                Representative.objects.values_list('full_name', 'pk')
            }

        full = (u'%s %s' % (prenom, nom)).replace(u' ', ' ')
36
37
38
39
40
41
        return self.deputes_slug.get(slugify(full), None)

    def get_depute_by_url(self, url):
        if self.deputes_rid is None:
            self.deputes_rid = {
                r[0]: r[1] for r in
42
43
44
                Representative.objects.prefetch_related('website_set')
                .filter(website__kind__in=['AN', 'SEN'])
                .values_list('website__url', 'pk')
45
46
47
            }

        return self.deputes_rid.get(url, None)
Nicolas Joyard's avatar
Nicolas Joyard committed
48
49
50
51
52
53
54
55
56
57
58

    def get_scrutin(self, ref):
        if self.scrutins is None:
            self.scrutins = {
                s[0]: s[1] for s in Proposal.objects.values_list('reference',
                                                                 'pk')
            }

        return self.scrutins.get(ref, None)

    def parse_vote_data(self, data):
59
        scrutin = self.get_scrutin(data['scrutin_url'])
Nicolas Joyard's avatar
Nicolas Joyard committed
60
61
        if scrutin is None:
            logger.debug('Cannot import vote for unknown scrutin %s'
62
                         % data['scrutin_url'])
Nicolas Joyard's avatar
Nicolas Joyard committed
63
64
            return

65
66
67
68
69
70
71
        if 'parl_url' in data:
            repdesc = data['parl_url']
            depute = self.get_depute_by_url(data['parl_url'])
        else:
            repdesc = '%s %s' % (data['prenom'], data['nom'])
            depute = self.get_depute_by_name(data['prenom'], data['nom'])

Nicolas Joyard's avatar
Nicolas Joyard committed
72
        if depute is None:
73
            logger.debug('Cannot import vote by unknown rep %s' % repdesc)
Nicolas Joyard's avatar
Nicolas Joyard committed
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
            return

        if not data['division'].lower() in self.positions:
            logger.debug('Cannot import vote for invalid position %s'
                         % data['division'])
            return
        position = self.positions[data['division'].lower()]

        changed = False
        try:
            vote = Vote.objects.get(representative_id=depute,
                                    proposal_id=scrutin)
        except Vote.DoesNotExist:
            vote = Vote(representative_id=depute, proposal_id=scrutin)
            logger.debug('Created vote for rep %s on %s' % (depute, scrutin))
            changed = True

        if vote.position != position:
            logger.debug('Changed vote position to %s' % position)
            changed = True
            vote.position = position

        if changed:
            logger.debug('Updated vote for rep %s on %s' % (depute, scrutin))
            self.touched.append(scrutin)
            vote.save()

    def update_totals(self):
        proposals = [Proposal.objects.get(pk=pk) for pk in self.touched]

        for proposal in proposals:
            changed = False

            for pos in self.positions.values():
                count = Vote.objects.filter(proposal_id=proposal.pk,
                                            position=pos).count()

                if getattr(proposal, 'total_%s' % pos, None) != count:
                    logger.debug('Changed %s count for proposal %s to %s' % (
                        pos, proposal.pk, count))
                    setattr(proposal, 'total_%s' % pos, count)
                    changed = True

            if changed:
                logger.debug('Updated proposal %s' % proposal.pk)
                proposal.save()


def main(stream=None):
    if not apps.ready:
        django.setup()

    importer = VotesImporter()

    for data in ijson.items(stream or sys.stdin, 'item'):
129
130
        try:
            importer.parse_vote_data(data)
131
132
        except Exception:
            logger.exception('error trying to import vote %s', str(data))
Nicolas Joyard's avatar
Nicolas Joyard committed
133
134

    importer.update_totals()