Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
luxcem
memopol
Commits
c061527a
Commit
c061527a
authored
Oct 11, 2016
by
Nicolas Joyard
Browse files
Import committee votes from parltrack
parent
ba69e530
Changes
2
Hide whitespace changes
Inline
Side-by-side
bin/update_votes
View file @
c061527a
...
...
@@ -7,6 +7,10 @@ parltrack_download_pipe ep_votes.json.xz parltrack_import_votes
sleep
10
parltrack_download_pipe ep_com_votes.json.xz parltrack_import_votes
sleep
10
francedata_download_pipe votes.json.gz francedata_import_votes
if
[
"x
$1
"
!=
"xnoscores"
]
;
then
...
...
src/representatives_votes/contrib/parltrack/import_votes.py
View file @
c061527a
...
...
@@ -28,6 +28,8 @@ def _parse_date(date_str):
JSON_URL
=
'http://parltrack.euwiki.org/dumps/ep_votes.json.xz'
DESTINATION
=
join
(
'/tmp'
,
'ep_votes.json'
)
RE_COMVOTE_REF
=
re
.
compile
(
r
'&reference=([^&]+)'
)
POSITION_MAP
=
{
k
:
k
for
k
in
(
'For'
,
'Against'
,
'Abstain'
)}
class
Command
(
object
):
...
...
@@ -61,11 +63,118 @@ class Command(object):
vote_data
[
'epref'
])
return
return
self
.
parse_proposal_data
(
proposal_data
=
vote_data
,
dossier_pk
=
dossier_pk
if
'committee'
in
vote_data
:
return
self
.
parse_committee_vote_data
(
vote_data
=
vote_data
,
dossier_pk
=
dossier_pk
)
else
:
return
self
.
parse_proposal_data
(
proposal_data
=
vote_data
,
dossier_pk
=
dossier_pk
)
def
parse_proposal_totals
(
self
,
data
,
position_map
=
POSITION_MAP
):
totals
=
{}
for
position
in
(
'For'
,
'Abstain'
,
'Against'
):
position_data
=
data
.
get
(
position_map
[
position
],
{})
position_total
=
position_data
.
get
(
'total'
,
0
)
if
isinstance
(
position_total
,
str
)
and
position_total
.
isdigit
():
position_total
=
int
(
position_total
)
totals
[
'total_%s'
%
position
.
lower
()]
=
position_total
return
totals
def
parse_proposal_votes
(
self
,
proposal
,
data
,
position_map
=
POSITION_MAP
):
logger
.
info
(
u
'Looking for votes in proposal {}'
.
format
(
proposal
.
title
))
for
position
in
(
'For'
,
'Abstain'
,
'Against'
):
for
group_vote_data
in
data
.
get
(
position
,
{}).
get
(
'groups'
,
{}):
for
vote_data
in
group_vote_data
[
'votes'
]:
if
not
isinstance
(
vote_data
,
dict
):
logger
.
error
(
'Skipping vote data %s for proposal %s'
,
vote_data
,
data
[
'_id'
])
continue
representative_pk
=
self
.
get_representative
(
vote_data
)
if
representative_pk
is
None
:
logger
.
error
(
'Could not find mep for %s'
,
vote_data
)
continue
changed
=
False
try
:
vote
=
Vote
.
objects
.
get
(
representative_id
=
representative_pk
,
proposal_id
=
proposal
.
pk
)
except
Vote
.
DoesNotExist
:
vote
=
Vote
(
proposal_id
=
proposal
.
pk
,
representative_id
=
representative_pk
)
changed
=
True
if
vote
.
position
!=
position
.
lower
():
changed
=
True
vote
.
position
=
position
.
lower
()
if
changed
:
vote
.
save
()
logger
.
debug
(
'Save vote %s for MEP %s on %s #%s to %s'
,
vote
.
pk
,
representative_pk
,
data
[
'title'
],
proposal
.
pk
,
position
)
@
transaction
.
atomic
def
parse_committee_vote_data
(
self
,
proposal_data
,
dossier_pk
):
title
=
u
'{} vote on {}'
%
(
proposal_data
[
'committee'
],
proposal_data
[
'doc'
])
changed
=
False
try
:
proposal
=
Proposal
.
objects
.
get
(
title
=
title
,
dossier_id
=
dossier_pk
)
except
Proposal
.
DoesNotExist
:
proposal
=
Proposal
(
title
=
title
,
dossier_id
=
dossier_pk
)
changed
=
True
try
:
ref
=
RE_COMVOTE_REF
.
search
(
proposal_data
[
'url'
]).
group
(
1
)
except
:
logger
.
debug
(
u
'Cannot find proposal reference for %s'
%
title
)
return
data_map
=
dict
(
datetime
=
_parse_date
(
proposal_data
[
'ts'
]),
reference
=
ref
,
kind
=
'Committee vote'
)
position_map
=
{
'For'
:
'+'
,
'Against'
:
'-'
,
'Abstain'
:
'0'
}
data_map
.
update
(
self
.
parse_proposal_totals
(
proposal_data
,
position_map
))
for
key
,
value
in
data_map
.
items
():
if
value
!=
getattr
(
proposal
,
key
,
None
):
setattr
(
proposal
,
key
,
value
)
changed
=
True
if
changed
:
proposal
.
save
()
if
self
.
should_skip
(
proposal_data
):
logger
.
debug
(
u
'Skipping votes for dossier %s'
,
proposal_data
.
get
(
'epref'
,
proposal_data
[
'title'
]))
return
self
.
parse_proposal_votes
(
proposal
,
proposal_data
,
position_map
)
@
transaction
.
atomic
def
parse_proposal_data
(
self
,
proposal_data
,
dossier_pk
):
"""Get or Create a proposal model from raw data"""
...
...
@@ -92,14 +201,7 @@ class Command(object):
kind
=
proposal_data
.
get
(
'issue_type'
)
)
for
position
in
(
'For'
,
'Abstain'
,
'Against'
):
position_data
=
proposal_data
.
get
(
position
,
{})
position_total
=
position_data
.
get
(
'total'
,
0
)
if
isinstance
(
position_total
,
str
)
and
position_total
.
isdigit
():
position_total
=
int
(
position_total
)
data_map
[
'total_%s'
%
position
.
lower
()]
=
position_total
data_map
.
update
(
self
.
parse_proposal_totals
(
proposal_data
))
for
key
,
value
in
data_map
.
items
():
if
value
!=
getattr
(
proposal
,
key
,
None
):
...
...
@@ -111,56 +213,11 @@ class Command(object):
if
self
.
should_skip
(
proposal_data
):
logger
.
debug
(
'Skipping dossier %s'
,
proposal_data
.
get
(
u
'Skipping
votes for
dossier %s'
,
proposal_data
.
get
(
'epref'
,
proposal_data
[
'title'
]))
return
positions
=
[
'For'
,
'Abstain'
,
'Against'
]
logger
.
info
(
'Looking for votes in proposal {}'
.
format
(
proposal_display
))
for
position
in
positions
:
for
group_vote_data
in
proposal_data
.
get
(
position
,
{}).
get
(
'groups'
,
{}):
for
vote_data
in
group_vote_data
[
'votes'
]:
if
not
isinstance
(
vote_data
,
dict
):
logger
.
error
(
'Skipping vote data %s for proposal %s'
,
vote_data
,
proposal_data
[
'_id'
])
continue
representative_pk
=
self
.
get_representative
(
vote_data
)
if
representative_pk
is
None
:
logger
.
error
(
'Could not find mep for %s'
,
vote_data
)
continue
representative_name
=
vote_data
.
get
(
'orig'
,
''
)
changed
=
False
try
:
vote
=
Vote
.
objects
.
get
(
representative_id
=
representative_pk
,
proposal_id
=
proposal
.
pk
)
except
Vote
.
DoesNotExist
:
vote
=
Vote
(
proposal_id
=
proposal
.
pk
,
representative_id
=
representative_pk
)
changed
=
True
if
vote
.
position
!=
position
.
lower
():
changed
=
True
vote
.
position
=
position
.
lower
()
if
vote
.
representative_name
!=
representative_name
:
changed
=
True
vote
.
representative_name
=
representative_name
if
changed
:
vote
.
save
()
logger
.
debug
(
'Save vote %s for MEP %s on %s #%s to %s'
,
vote
.
pk
,
representative_pk
,
proposal_data
[
'title'
],
proposal
.
pk
,
position
)
self
.
parse_proposal_votes
(
proposal
,
proposal_data
)
return
proposal
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment