Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
memopol
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Porkepix
memopol
Commits
c061527a
Commit
c061527a
authored
Oct 11, 2016
by
Nicolas Joyard
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Import committee votes from parltrack
parent
ba69e530
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
119 additions
and
58 deletions
+119
-58
bin/update_votes
bin/update_votes
+4
-0
src/representatives_votes/contrib/parltrack/import_votes.py
src/representatives_votes/contrib/parltrack/import_votes.py
+115
-58
No files found.
bin/update_votes
View file @
c061527a
...
...
@@ -7,6 +7,10 @@ parltrack_download_pipe ep_votes.json.xz parltrack_import_votes
sleep
10
parltrack_download_pipe ep_com_votes.json.xz parltrack_import_votes
sleep
10
francedata_download_pipe votes.json.gz francedata_import_votes
if
[
"x
$1
"
!=
"xnoscores"
]
;
then
...
...
src/representatives_votes/contrib/parltrack/import_votes.py
View file @
c061527a
...
...
@@ -28,6 +28,8 @@ def _parse_date(date_str):
JSON_URL
=
'http://parltrack.euwiki.org/dumps/ep_votes.json.xz'
DESTINATION
=
join
(
'/tmp'
,
'ep_votes.json'
)
RE_COMVOTE_REF
=
re
.
compile
(
r
'&reference=([^&]+)'
)
POSITION_MAP
=
{
k
:
k
for
k
in
(
'For'
,
'Against'
,
'Abstain'
)}
class
Command
(
object
):
...
...
@@ -61,11 +63,118 @@ class Command(object):
vote_data
[
'epref'
])
return
return
self
.
parse_proposal_data
(
proposal_data
=
vote_data
,
dossier_pk
=
dossier_pk
if
'committee'
in
vote_data
:
return
self
.
parse_committee_vote_data
(
vote_data
=
vote_data
,
dossier_pk
=
dossier_pk
)
else
:
return
self
.
parse_proposal_data
(
proposal_data
=
vote_data
,
dossier_pk
=
dossier_pk
)
def
parse_proposal_totals
(
self
,
data
,
position_map
=
POSITION_MAP
):
totals
=
{}
for
position
in
(
'For'
,
'Abstain'
,
'Against'
):
position_data
=
data
.
get
(
position_map
[
position
],
{})
position_total
=
position_data
.
get
(
'total'
,
0
)
if
isinstance
(
position_total
,
str
)
and
position_total
.
isdigit
():
position_total
=
int
(
position_total
)
totals
[
'total_%s'
%
position
.
lower
()]
=
position_total
return
totals
def
parse_proposal_votes
(
self
,
proposal
,
data
,
position_map
=
POSITION_MAP
):
logger
.
info
(
u
'Looking for votes in proposal {}'
.
format
(
proposal
.
title
))
for
position
in
(
'For'
,
'Abstain'
,
'Against'
):
for
group_vote_data
in
data
.
get
(
position
,
{}).
get
(
'groups'
,
{}):
for
vote_data
in
group_vote_data
[
'votes'
]:
if
not
isinstance
(
vote_data
,
dict
):
logger
.
error
(
'Skipping vote data %s for proposal %s'
,
vote_data
,
data
[
'_id'
])
continue
representative_pk
=
self
.
get_representative
(
vote_data
)
if
representative_pk
is
None
:
logger
.
error
(
'Could not find mep for %s'
,
vote_data
)
continue
changed
=
False
try
:
vote
=
Vote
.
objects
.
get
(
representative_id
=
representative_pk
,
proposal_id
=
proposal
.
pk
)
except
Vote
.
DoesNotExist
:
vote
=
Vote
(
proposal_id
=
proposal
.
pk
,
representative_id
=
representative_pk
)
changed
=
True
if
vote
.
position
!=
position
.
lower
():
changed
=
True
vote
.
position
=
position
.
lower
()
if
changed
:
vote
.
save
()
logger
.
debug
(
'Save vote %s for MEP %s on %s #%s to %s'
,
vote
.
pk
,
representative_pk
,
data
[
'title'
],
proposal
.
pk
,
position
)
@
transaction
.
atomic
def
parse_committee_vote_data
(
self
,
proposal_data
,
dossier_pk
):
title
=
u
'{} vote on {}'
%
(
proposal_data
[
'committee'
],
proposal_data
[
'doc'
])
changed
=
False
try
:
proposal
=
Proposal
.
objects
.
get
(
title
=
title
,
dossier_id
=
dossier_pk
)
except
Proposal
.
DoesNotExist
:
proposal
=
Proposal
(
title
=
title
,
dossier_id
=
dossier_pk
)
changed
=
True
try
:
ref
=
RE_COMVOTE_REF
.
search
(
proposal_data
[
'url'
]).
group
(
1
)
except
:
logger
.
debug
(
u
'Cannot find proposal reference for %s'
%
title
)
return
data_map
=
dict
(
datetime
=
_parse_date
(
proposal_data
[
'ts'
]),
reference
=
ref
,
kind
=
'Committee vote'
)
position_map
=
{
'For'
:
'+'
,
'Against'
:
'-'
,
'Abstain'
:
'0'
}
data_map
.
update
(
self
.
parse_proposal_totals
(
proposal_data
,
position_map
))
for
key
,
value
in
data_map
.
items
():
if
value
!=
getattr
(
proposal
,
key
,
None
):
setattr
(
proposal
,
key
,
value
)
changed
=
True
if
changed
:
proposal
.
save
()
if
self
.
should_skip
(
proposal_data
):
logger
.
debug
(
u
'Skipping votes for dossier %s'
,
proposal_data
.
get
(
'epref'
,
proposal_data
[
'title'
]))
return
self
.
parse_proposal_votes
(
proposal
,
proposal_data
,
position_map
)
@
transaction
.
atomic
def
parse_proposal_data
(
self
,
proposal_data
,
dossier_pk
):
"""Get or Create a proposal model from raw data"""
...
...
@@ -92,14 +201,7 @@ class Command(object):
kind
=
proposal_data
.
get
(
'issue_type'
)
)
for
position
in
(
'For'
,
'Abstain'
,
'Against'
):
position_data
=
proposal_data
.
get
(
position
,
{})
position_total
=
position_data
.
get
(
'total'
,
0
)
if
isinstance
(
position_total
,
str
)
and
position_total
.
isdigit
():
position_total
=
int
(
position_total
)
data_map
[
'total_%s'
%
position
.
lower
()]
=
position_total
data_map
.
update
(
self
.
parse_proposal_totals
(
proposal_data
))
for
key
,
value
in
data_map
.
items
():
if
value
!=
getattr
(
proposal
,
key
,
None
):
...
...
@@ -111,56 +213,11 @@ class Command(object):
if
self
.
should_skip
(
proposal_data
):
logger
.
debug
(
'Skipping
dossier %s'
,
proposal_data
.
get
(
u
'Skipping votes for
dossier %s'
,
proposal_data
.
get
(
'epref'
,
proposal_data
[
'title'
]))
return
positions
=
[
'For'
,
'Abstain'
,
'Against'
]
logger
.
info
(
'Looking for votes in proposal {}'
.
format
(
proposal_display
))
for
position
in
positions
:
for
group_vote_data
in
proposal_data
.
get
(
position
,
{}).
get
(
'groups'
,
{}):
for
vote_data
in
group_vote_data
[
'votes'
]:
if
not
isinstance
(
vote_data
,
dict
):
logger
.
error
(
'Skipping vote data %s for proposal %s'
,
vote_data
,
proposal_data
[
'_id'
])
continue
representative_pk
=
self
.
get_representative
(
vote_data
)
if
representative_pk
is
None
:
logger
.
error
(
'Could not find mep for %s'
,
vote_data
)
continue
representative_name
=
vote_data
.
get
(
'orig'
,
''
)
changed
=
False
try
:
vote
=
Vote
.
objects
.
get
(
representative_id
=
representative_pk
,
proposal_id
=
proposal
.
pk
)
except
Vote
.
DoesNotExist
:
vote
=
Vote
(
proposal_id
=
proposal
.
pk
,
representative_id
=
representative_pk
)
changed
=
True
if
vote
.
position
!=
position
.
lower
():
changed
=
True
vote
.
position
=
position
.
lower
()
if
vote
.
representative_name
!=
representative_name
:
changed
=
True
vote
.
representative_name
=
representative_name
if
changed
:
vote
.
save
()
logger
.
debug
(
'Save vote %s for MEP %s on %s #%s to %s'
,
vote
.
pk
,
representative_pk
,
proposal_data
[
'title'
],
proposal
.
pk
,
position
)
self
.
parse_proposal_votes
(
proposal
,
proposal_data
)
return
proposal
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment