import_recommendations.py 3.95 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
# coding: utf-8

import csv
import django
from django.apps import apps
import logging
import sys

from representatives_recommendations.models import Recommendation
from representatives_votes.models import Dossier, Proposal

12
from .import_data import dossier_mappings, resolutions
13

Nicolas Joyard's avatar
Nicolas Joyard committed
14
logger = logging.getLogger(__name__)
15
16
17
18
19
20
21
22
23
24


class RecommendationImporter:
    def __init__(self):
        self.dossier_cache = {}

    def get_dossier(self, title):
        dossier = self.dossier_cache.get(title, None)

        if dossier is None:
Nicolas Joyard's avatar
Nicolas Joyard committed
25
26
            ref = dossier_mappings.get(title, None)
            if ref is not None:
27
                query = {'reference': ref}
Nicolas Joyard's avatar
Nicolas Joyard committed
28
            else:
29
                query = {'title__iexact': title}
Nicolas Joyard's avatar
Nicolas Joyard committed
30

31
            try:
Nicolas Joyard's avatar
Nicolas Joyard committed
32
                dossier = Dossier.objects.get(**query)
33
                self.dossier_cache[title] = dossier
34
35
36
37
38
39
            except Dossier.DoesNotExist:
                dossier = None

        return dossier

    def get_proposal(self, dossier, kind):
40
41
        kinds = [kind]

42
        try:
43
44
45
46
47
48
49
50
51
52
53
54
            resolutions.index(kind.lower())
            kinds.extend(resolutions)
        except ValueError:
            pass

        for k in kinds:
            try:
                return Proposal.objects.get(dossier=dossier, kind__iexact=k)
            except Proposal.DoesNotExist:
                continue

        return None
55
56

    def import_row(self, row):
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
        try:
            dossier = self.get_dossier(row['title'])
            if dossier is None:
                logger.warn('No dossier "%s"' % row['title'])
                return False

            proposal = self.get_proposal(dossier, row['part'])
            if proposal is None:
                logger.warn('No proposal "%s" for dossier %s (%d): "%s"' % (
                    row['part'].decode('utf-8'), dossier.reference, dossier.pk,
                    row['title']))
                return False

            weight = int(row['weight']) * int(row['ponderation'])
            descr = row['description'].strip()
            if len(descr) == 0:
                descr = '%s on %s' % (row['part'], dossier.reference)
74

75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
            try:
                recom = Recommendation.objects.get(proposal=proposal)
            except Recommendation.DoesNotExist:
                recom = Recommendation(
                    proposal=proposal,
                    recommendation=row['recommendation'],
                    title=descr,
                    weight=weight
                )
                recom.save()
                logger.info('Created recommendation with weight %s for %s: %s'
                            % (
                                weight,
                                row['title'],
                                row['part']
                            ))

            return True
93
94
95
        except Exception:
            logger.exception('error trying to import recommendation %s',
                             str(row))
96
97
98
99
100
101
102
103
            return False


def main(stream=None):
    """
    Imports recommendations from an old memopol instance.

    Usage:
104
        cat recommendations.csv | memopol_import_recommendations
105
106

    The input CSV file should be generated by the following query:
107
108
        SELECT CONCAT(r.description, '|', r.weight, '|', r.recommendation, '|',
            r.part, '|', p.title, '|', p.ponderation)
109
110
111
112
113
114
115
116
117
118
119
120
121
        FROM votes_recommendation r
            LEFT JOIN votes_proposal p ON r.proposal_id = p.id
        WHERE p.institution = 'EU'

    """

    if not apps.ready:
        django.setup()

    importer = RecommendationImporter()
    rejected = []
    imported = 0

122
    reader = csv.DictReader(stream or sys.stdin, delimiter='|', fieldnames=[
123
124
125
126
        'description',
        'weight',
        'recommendation',
        'part',
127
128
        'title',
        'ponderation'
129
130
131
132
133
134
135
136
137
    ], quoting=csv.QUOTE_NONE)

    for row in reader:
        if not importer.import_row(row):
            rejected.append(row)
        else:
            imported = imported + 1

    logger.info('%d rows imported, %d rows rejected', imported, len(rejected))