import_scrutins.py 2.97 KB
Newer Older
Nicolas Joyard's avatar
Nicolas Joyard committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
# coding: utf-8

from datetime import datetime
import ijson
import logging
from pytz import timezone as date_timezone
import sys

import django
from django.apps import apps
from django.utils.timezone import make_aware as date_make_aware

from representatives_votes.models import Dossier, Proposal

logger = logging.getLogger(__name__)


def _parse_date(date_str):
    return date_make_aware(
        datetime.strptime(date_str, "%Y-%m-%d"),
        date_timezone('Europe/Paris')
    )


def _get_unique_title(proposal_pk, candidate):
    title = candidate

    try:
        exists = Proposal.objects.get(title=title)
    except Proposal.DoesNotExist:
        exists = None

    if exists and exists.pk != proposal_pk:
        num = 1
        while exists and exists.pk != proposal_pk:
            title = '%s (%d)' % (candidate, num)

            try:
                exists = Proposal.objects.get(title=title)
            except Proposal.DoesNotExist:
                exists = None

            num = num + 1

        logger.debug('Made unique title %s' % title)

    return title


class ScrutinImporter:
51
    dossiers = {}
Nicolas Joyard's avatar
Nicolas Joyard committed
52

53
    def get_dossier(self, url):
54 55 56 57 58
        if url not in self.dossiers:
            try:
                self.dossiers[url] = Dossier.objects.get(documents__link=url)
            except Dossier.DoesNotExist:
                return None
59

60
        return self.dossiers[url]
Nicolas Joyard's avatar
Nicolas Joyard committed
61 62

    def parse_scrutin_data(self, data):
63
        ref = data['url']
Nicolas Joyard's avatar
Nicolas Joyard committed
64

65
        if 'dossier_url' not in data:
Nicolas Joyard's avatar
Nicolas Joyard committed
66 67 68
            logger.debug('Cannot create proposal without dossier')
            return

69
        dossier = self.get_dossier(data['dossier_url'])
Nicolas Joyard's avatar
Nicolas Joyard committed
70 71
        if dossier is None:
            logger.debug('Cannot create proposal for unknown dossier %s'
72
                         % data['dossier_url'])
Nicolas Joyard's avatar
Nicolas Joyard committed
73 74 75 76 77 78 79 80 81 82 83 84 85 86
            return

        changed = False
        try:
            proposal = Proposal.objects.get(reference=ref)
        except Proposal.DoesNotExist:
            proposal = Proposal(reference=ref, total_for=0, total_against=0,
                                total_abstain=0)
            logger.debug('Created proposal %s' % ref)
            changed = True

        values = dict(
            title=_get_unique_title(proposal.pk, data["objet"]),
            datetime=_parse_date(data["date"]),
87
            dossier_id=dossier.pk,
Nicolas Joyard's avatar
Nicolas Joyard committed
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
            kind='dossier'
        )

        for key, value in values.items():
            if value != getattr(proposal, key, None):
                logger.debug('Changed proposal %s to %s' % (key, value))
                setattr(proposal, key, value)
                changed = True

        if changed:
            logger.debug('Updated proposal %s' % ref)
            proposal.save()


def main(stream=None):
    if not apps.ready:
        django.setup()

    importer = ScrutinImporter()

    for data in ijson.items(stream or sys.stdin, 'item'):
109 110
        try:
            importer.parse_scrutin_data(data)
111 112
        except Exception:
            logger.exception('error trying to import scrutin %s', str(data))