Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
C
campaign
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
La Quadrature du Net
piphone
campaign
Commits
66600852
Commit
66600852
authored
Aug 09, 2016
by
okhin
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
WIP
#5
: Representative importer done, along with the new category field inthe model
parent
e77669eb
Pipeline
#102
failed with stage
in 50 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
101 additions
and
76 deletions
+101
-76
picampaign/importer/admin.py
picampaign/importer/admin.py
+1
-1
picampaign/importer/migrations/0002_auto_20160809_1442.py
picampaign/importer/migrations/0002_auto_20160809_1442.py
+25
-0
picampaign/importer/models.py
picampaign/importer/models.py
+75
-75
No files found.
picampaign/importer/admin.py
View file @
66600852
...
...
@@ -4,7 +4,7 @@ from django.utils.translation import ugettext_lazy as _
from
picampaign.importer.models
import
Importer
class
ImporterAdmin
(
admin
.
ModelAdmin
):
list_display
=
[
'name'
,
'kind'
,
'
data_format'
,
'file'
,
'url
'
]
list_display
=
[
'name'
,
'kind'
,
'
category
'
]
actions
=
[
'import_action'
]
def
import_action
(
self
,
request
,
queryset
):
...
...
picampaign/importer/migrations/0002_auto_20160809_1442.py
0 → 100644
View file @
66600852
# -*- coding: utf-8 -*-
# Generated by Django 1.9.6 on 2016-08-09 12:42
from
__future__
import
unicode_literals
from
django.db
import
migrations
,
models
class
Migration
(
migrations
.
Migration
):
dependencies
=
[
(
'importer'
,
'0001_initial'
),
]
operations
=
[
migrations
.
RemoveField
(
model_name
=
'importer'
,
name
=
'data_format'
,
),
migrations
.
AddField
(
model_name
=
'importer'
,
name
=
'category'
,
field
=
models
.
CharField
(
choices
=
[(
'update'
,
'CSV Updater'
),
(
'repr'
,
'Representative format'
)],
default
=
'repr'
,
max_length
=
6
),
preserve_default
=
False
,
),
]
picampaign/importer/models.py
View file @
66600852
...
...
@@ -14,7 +14,8 @@ class Importer(models.Model):
"""Importer model. Used to populate campaign with contacts"""
id
=
models
.
AutoField
(
primary_key
=
True
)
name
=
models
.
CharField
(
max_length
=
64
)
data_format
=
models
.
CharField
(
max_length
=
4
,
choices
=
[(
'JSON'
,
'json'
),
(
'CSV'
,
'csv'
)])
category
=
models
.
CharField
(
max_length
=
6
,
choices
=
((
'update'
,
_
(
'CSV Updater'
))
,
(
'repr'
,
_
(
"Representative format"
))))
file
=
models
.
FileField
(
upload_to
=
'imports/'
,
blank
=
True
,
null
=
True
)
url
=
models
.
URLField
(
blank
=
True
,
null
=
True
)
campaign
=
models
.
ForeignKey
(
Campaign
,
on_delete
=
models
.
SET_NULL
,
null
=
True
)
...
...
@@ -23,7 +24,7 @@ class Importer(models.Model):
last_imported
=
models
.
IntegerField
(
null
=
True
,
blank
=
True
)
def
__str__
(
self
):
return
_
(
u
'%(name)s: %(format)s %(type)s importer'
)
%
{
'format'
:
self
.
data_format
,
return
_
(
u
'%(name)s: %(format)s %(type)s importer'
)
%
{
'format'
:
self
.
category
,
'type'
:
self
.
kind
(),
'name'
:
self
.
name
}
...
...
@@ -35,56 +36,24 @@ class Importer(models.Model):
else
:
return
'url'
def
handle
(
self
):
def
representative
(
self
,
imported_data
):
"""
This function is used to import the data described by the Importer.
We need to parse the provided file, using an appropriate deserilizer/parser,
and then to create or update contacts.
The operation should end providing the number of items imported and the number of items submitted.
Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
holding a list of groups (mandates) of a specific kind, column separated
For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
by columns.
This handle all the representatives data import. The imported_data have all the contacts in
the representative format (and in a list).
"""
import_data
=
[]
# First let's check what kind of importer are we running.
# If it's a file, let's parse it according to our format.
if
self
.
kind
()
==
'file'
:
with
open
(
self
.
file
.
path
,
u
'rU'
)
as
f
:
if
self
.
data_format
==
'CSV'
:
# If we're a csv format, let's use DictReader on a file object
reader
=
DictReader
(
f
)
re_mandate
=
re
.
compile
(
r
'^mandate:(\w+)'
)
for
item
in
reader
:
# We need to create the groups according to the header
for
key
in
item
.
keys
():
if
re
.
match
(
re_mandate
,
key
):
# We have a mandate
for
mandate
in
item
[
key
].
split
(
':'
):
item
[
'mandates'
].
append
({
'group'
:
{
'kind'
:
key
.
split
(
':'
)[
0
],
'name'
:
mandate
}})
item
.
delete
(
key
)
import_data
.
append
(
item
)
else
:
# We're in json, let's load from a file
import_data
=
json
.
load
(
f
)
# We now have all the data ready to be imported.
# We can count the data available
self
.
last_count
=
len
(
import_data
)
# Lets' go through the meps, for fun and profit
inserted
=
0
for
import_
mep
in
import_data
:
for
import_
contact
in
import_data
:
# We want to know if all mandates exists
if
not
'birth_date'
in
import_
mep
or
not
'first_name'
in
import_mep
or
not
'last_name'
in
import_mep
:
# We do not have enough field to discriminate our
mep
s
if
not
'birth_date'
in
import_
contact
or
not
'first_name'
in
import_contact
or
not
'last_name'
in
import_contact
:
# We do not have enough field to discriminate our
contact
s
continue
updated_
mep
=
{}
updated_
contact
=
{}
groups
=
[]
for
mandate
in
import_
mep
[
'mandates'
]:
for
mandate
in
import_
contact
[
'mandates'
]:
# Parltrack gives us a date starting with 9999 if ends has not happens
# FranceData gives us an empty one
end_date
=
mandate
.
get
(
'end_date'
,
None
)
...
...
@@ -108,52 +77,83 @@ class Importer(models.Model):
type
=
groupType
)
groups
.
append
(
group
)
if
groups
==
[]:
# The
mep
have no groups active, he is not to be bothered.
# The
contact
have no groups active, he is not to be bothered.
continue
# All groups are done
# If we have a contact item, we can go through it. Otherwise we probably have
# phone, twitter, etc fields.
phones
=
[]
if
'contacts'
in
import_mep
:
# Let's get the first phone
if
import_mep
[
'contacts'
][
'phones'
]
!=
[]:
for
phone
in
import_mep
[
'contacts'
][
'phones'
]:
if
phone
[
'kind'
]
==
"office phone"
:
phones
.
append
(
phone
[
'number'
])
if
import_mep
[
'contacts'
][
'emails'
]
!=
[]:
updated_mep
[
'mail'
]
=
import_mep
[
'contacts'
][
'emails'
][
0
][
'email'
]
if
import_mep
[
'contacts'
][
'websites'
]
!=
[]:
for
website
in
import_mep
[
'contacts'
][
'websites'
]:
if
website
[
'kind'
]
==
'twitter'
:
updated_mep
[
'twitter'
]
=
''
.
join
([
'@'
,
website
[
'url'
].
split
(
'/'
)[
-
1
]])
break
else
:
# We have phone, mail, twitter fields
if
'phone'
in
import_mep
:
phones
.
append
(
import_mep
[
'phone'
])
if
'mail'
in
import_mep
:
updated_mep
[
'mail'
]
=
import_mep
[
'phone'
]
if
'twitter'
in
import_mep
:
updated_mep
[
'twitter'
]
=
import_mep
[
'twitter'
]
# Let's update_or_create the mep
mep
,
updated
=
Contact
.
objects
.
update_or_create
(
first_name
=
import_mep
[
'first_name'
],
last_name
=
import_mep
[
'last_name'
],
birthdate
=
import_mep
[
'birth_date'
],
defaults
=
updated_mep
# Let's get the phones
if
import_contact
[
'contacts'
][
'phones'
]
!=
[]:
for
phone
in
import_contact
[
'contacts'
][
'phones'
]:
if
phone
[
'kind'
]
==
"office phone"
:
phones
.
append
(
phone
[
'number'
])
# These are the email adresses used
if
import_contact
[
'contacts'
][
'emails'
]
!=
[]:
updated_contact
[
'mail'
]
=
import_contact
[
'contacts'
][
'emails'
][
0
][
'email'
]
# We're trying to find the twitter accounts
if
import_contact
[
'contacts'
][
'websites'
]
!=
[]:
for
website
in
import_contact
[
'contacts'
][
'websites'
]:
if
website
[
'kind'
]
==
'twitter'
:
updated_contact
[
'twitter'
]
=
''
.
join
([
'@'
,
website
[
'url'
].
split
(
'/'
)[
-
1
]])
break
# Let's update_or_create the contact
contact
,
updated
=
Contact
.
objects
.
update_or_create
(
first_name
=
import_contact
[
'first_name'
],
last_name
=
import_contact
[
'last_name'
],
birthdate
=
import_contact
[
'birth_date'
],
defaults
=
updated_contact
)
# We want to create the phones
for
phone
in
phones
:
mep
.
phones
.
get_or_create
(
phone
=
phone
)
contact
.
phones
.
get_or_create
(
phone
=
phone
)
# Let's link the MEP to the group
mep
.
groups
=
groups
mep
.
save
()
# We want to link the
mep
to our current campaign
CampaignContact
(
campaign
=
self
.
campaign
,
contact
=
mep
).
save
()
contact
.
groups
=
groups
contact
.
save
()
# We want to link the
contact
to our current campaign
CampaignContact
(
campaign
=
self
.
campaign
,
contact
=
contact
).
save
()
inserted
+=
1
# Updating the last imported item
self
.
last_imported
=
inserted
self
.
save
()
def
handle
(
self
):
"""
This function is used to import the data described by the Importer.
We need to parse the provided file, using an appropriate deserilizer/parser,
and then to create or update contacts.
The operation should end providing the number of items imported and the number of items submitted.
Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
holding a list of groups (mandates) of a specific kind, column separated
For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
by columns.
"""
import_data
=
[]
# First let's check what kind of importer are we running.
# If it's a file, let's parse it according to our format.
if
self
.
kind
()
==
'file'
:
with
open
(
self
.
file
.
path
,
u
'rU'
)
as
f
:
if
self
.
data_format
==
'CSV'
:
# If we're a csv format, let's use DictReader on a file object
reader
=
DictReader
(
f
)
for
item
in
reader
:
# We need to create the groups according to the header
import_data
.
append
(
item
)
else
:
# We're in json, let's load from a file
import_data
=
json
.
load
(
f
)
# Now, let's get the correct call
if
self
.
category
==
'repr'
:
self
.
representative
(
import_data
)
return
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment