import_scrutins.py 2.97 KB
Newer Older
Nicolas Joyard's avatar
Nicolas Joyard committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# coding: utf-8

from datetime import datetime
import ijson
import logging
from pytz import timezone as date_timezone
import sys

import django
from django.apps import apps
from django.utils.timezone import make_aware as date_make_aware

from representatives_votes.models import Dossier, Proposal

logger = logging.getLogger(__name__)


def _parse_date(date_str):
    return date_make_aware(
        datetime.strptime(date_str, "%Y-%m-%d"),
        date_timezone('Europe/Paris')
    )


def _get_unique_title(proposal_pk, candidate):
    title = candidate

    try:
        exists = Proposal.objects.get(title=title)
    except Proposal.DoesNotExist:
        exists = None

    if exists and exists.pk != proposal_pk:
        num = 1
        while exists and exists.pk != proposal_pk:
            title = '%s (%d)' % (candidate, num)

            try:
                exists = Proposal.objects.get(title=title)
            except Proposal.DoesNotExist:
                exists = None

            num = num + 1

        logger.debug('Made unique title %s' % title)

    return title


class ScrutinImporter:
51
    dossiers = {}
Nicolas Joyard's avatar
Nicolas Joyard committed
52

53
    def get_dossier(self, url):
54
55
56
57
58
        if url not in self.dossiers:
            try:
                self.dossiers[url] = Dossier.objects.get(documents__link=url)
            except Dossier.DoesNotExist:
                return None
59

60
        return self.dossiers[url]
Nicolas Joyard's avatar
Nicolas Joyard committed
61
62

    def parse_scrutin_data(self, data):
63
        ref = data['url']
Nicolas Joyard's avatar
Nicolas Joyard committed
64

65
        if 'dossier_url' not in data:
Nicolas Joyard's avatar
Nicolas Joyard committed
66
67
68
            logger.debug('Cannot create proposal without dossier')
            return

69
        dossier = self.get_dossier(data['dossier_url'])
Nicolas Joyard's avatar
Nicolas Joyard committed
70
71
        if dossier is None:
            logger.debug('Cannot create proposal for unknown dossier %s'
72
                         % data['dossier_url'])
Nicolas Joyard's avatar
Nicolas Joyard committed
73
74
75
76
77
78
79
80
81
82
83
84
85
86
            return

        changed = False
        try:
            proposal = Proposal.objects.get(reference=ref)
        except Proposal.DoesNotExist:
            proposal = Proposal(reference=ref, total_for=0, total_against=0,
                                total_abstain=0)
            logger.debug('Created proposal %s' % ref)
            changed = True

        values = dict(
            title=_get_unique_title(proposal.pk, data["objet"]),
            datetime=_parse_date(data["date"]),
87
            dossier_id=dossier.pk,
Nicolas Joyard's avatar
Nicolas Joyard committed
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
            kind='dossier'
        )

        for key, value in values.items():
            if value != getattr(proposal, key, None):
                logger.debug('Changed proposal %s to %s' % (key, value))
                setattr(proposal, key, value)
                changed = True

        if changed:
            logger.debug('Updated proposal %s' % ref)
            proposal.save()


def main(stream=None):
    if not apps.ready:
        django.setup()

    importer = ScrutinImporter()

    for data in ijson.items(stream or sys.stdin, 'item'):
109
110
        try:
            importer.parse_scrutin_data(data)
111
112
        except Exception:
            logger.exception('error trying to import scrutin %s', str(data))