Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
La Quadrature du Net
piphone
campaign
Commits
ff7abd46
Commit
ff7abd46
authored
Aug 07, 2016
by
okhin
Browse files
FIrst run of the importer
parent
80367ef2
Changes
3
Hide whitespace changes
Inline
Side-by-side
picampaign/importer/admin.py
View file @
ff7abd46
from
django.contrib
import
admin
from
django.utils.translation
import
ugettext_lazy
as
_
from
picampaign.importer.models
import
Importer
class
ImporterAdmin
(
admin
.
ModelAdmin
):
list_display
=
[
'name'
,
'kind'
,
'data_format'
,
'file'
,
'url'
]
actions
=
[
'import_action'
]
admin
.
site
.
register
(
Importer
,
ImporterAdmin
)
def
import_action
(
self
,
request
,
queryset
):
"""
We're going to launch the import of data
"""
for
obj
in
queryset
:
obj
.
handle
()
self
.
message_user
(
request
,
_
(
"Imported %(last_imported)s of %(last_imported)s contacts"
)
%
{
'last_imported'
:
obj
.
last_imported
,
'last_count'
:
obj
.
last_count
})
import_action
.
short_description
=
_
(
"Run the selected importers"
)
admin
.
site
.
register
(
Importer
,
ImporterAdmin
)
picampaign/importer/models.py
View file @
ff7abd46
from
csv
import
DictReader
import
json
import
requests
import
re
from
django.db
import
models
from
django.utils.translation
import
ugettext_lazy
as
_
from
picampaign.campaign.models
import
Campaign
from
picampaign.organization.models
import
Group
,
GroupType
,
Organization
class
Importer
(
models
.
Model
):
"""Importer model. Used to populate campaign with contacts"""
...
...
@@ -11,11 +17,14 @@ class Importer(models.Model):
file
=
models
.
FileField
(
upload_to
=
'imports/'
,
blank
=
True
,
null
=
True
)
url
=
models
.
URLField
(
blank
=
True
,
null
=
True
)
campaign
=
models
.
ForeignKey
(
Campaign
,
on_delete
=
models
.
SET_NULL
,
null
=
True
)
organization
=
models
.
ForeignKey
(
Organization
,
on_delete
=
models
.
CASCADE
)
last_count
=
models
.
IntegerField
(
null
=
True
,
blank
=
True
)
last_imported
=
models
.
IntegerField
(
null
=
True
,
blank
=
True
)
def
__str__
(
self
):
return
_
(
'%
{
name
}
: %
{
format
}
%
{
type
}
importer'
%
{
'format'
:
self
.
data_format
,
return
_
(
u
'%
(
name
)s
: %
(
format
)s
%
(
type
)s
importer'
)
%
{
'format'
:
self
.
data_format
,
'type'
:
self
.
kind
(),
'name'
:
self
.
name
}
)
'name'
:
self
.
name
}
def
kind
(
self
):
if
self
.
url
is
None
and
self
.
file
is
None
:
...
...
@@ -24,3 +33,95 @@ class Importer(models.Model):
return
'file'
else
:
return
'url'
def
handle
(
self
):
"""
This function is used to import the data described by the Importer.
We need to parse the provided file, using an appropriate deserilizer/parser,
and then to create or update contacts.
The operation should end providing the number of items imported and the number of items submitted.
Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
holding a list of groups (mandates) of a specific kind, column separated
For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
by columns.
"""
import_data
=
[]
# First let's check what kind of importer are we running.
# If it's a file, let's parse it according to our format.
if
self
.
kind
==
'file'
:
with
open
(
self
.
file
,
u
'rU'
)
as
f
:
if
self
.
data_format
==
'CSV'
:
# If we're a csv format, let's use DictReader on a file object
reader
=
DictReader
(
f
)
re_mandate
=
re
.
compile
(
r
'^mandate:(\w+)'
)
for
item
in
reader
:
# We need to create the groups according to the header
for
key
in
item
.
keys
():
if
re
.
match
(
re_mandate
,
key
):
# We have a mandate
for
mandate
in
item
[
key
].
split
(
':'
):
item
[
'mandates'
].
append
({
'kind'
:
key
.
split
(
':'
)[
0
],
'name'
:
mandate
})
item
.
delete
(
key
)
import_data
.
append
(
item
)
else
:
# We're in json, let's load from a file
import_data
=
json
.
load
(
f
)
# We now have all the data ready to be imported.
# We can count the data available
self
.
last_count
=
len
(
import_data
)
# Lets' go through the meps, for fun and profit
inserted
=
0
for
import_mep
in
import_data
:
# We want to know if all mandates exists
if
not
'birthdate'
in
import_mep
or
not
'first_name'
in
import_mep
or
not
'last_name'
in
import_mep
:
# We do not have enough field to discriminate our meps
continue
updated_mep
=
{}
updated_mep
[
'groups'
]
=
[]
for
mandate
in
import_mep
[
'mandates'
]:
if
mandate
[
'end_date'
]
!=
''
:
# This mandate has ended
continue
groupType
,
added
=
GroupType
.
objects
.
get_or_create
(
organization
=
self
.
organization
,
name
=
mandate
[
'kind'
])
group
,
added
=
Group
.
objects
.
get_or_create
(
name
=
mandate
[
'name'
],
type
=
groupType
)
updated_mep
[
'groups'
].
append
(
group
)
# If we have a contact item, we can go through it. Otherwise we probably have
# phone, twitter, etc fields.
if
'contacts'
in
import_mep
:
# Let's get the first phone
if
import_mep
[
'contacts'
][
'phones'
]
!=
[]:
updated_emp
[
'phone'
]
=
mep
[
'contacts'
][
'phones'
][
0
][
'phone'
]
if
import_mep
[
'contacts'
][
'emails'
]
!=
[]:
updated_mep
[
'mail'
]
=
mep
[
'contacts'
][
'phones'
][
0
][
'phone'
]
if
import_mep
[
'contacts'
][
'websites'
]
!=
[]:
for
website
in
import_mep
[
'contacts'
][
'websites'
]:
if
website
[
'kind'
]
==
'twitter'
:
updated_mep
[
'twitter'
]
=
''
.
join
([
'@'
,
website
[
'website'
].
split
(
'/'
)[
-
1
]])
break
else
:
# We have phone, mail, twitter fields
if
'phone'
in
import_mep
:
updated_mep
[
'phone'
]
=
import_mep
[
'phone'
]
if
'mail'
in
import_mep
:
updated_mep
[
'mail'
]
=
import_mep
[
'phone'
]
if
'twitter'
in
import_mep
:
updated_mep
[
'twitter'
]
=
import_mep
[
'twitter'
]
# All groups are done
# Let's update_or_create the mep
mep
,
updated
=
Contact
.
objects
.
update_or_create
(
first_name
=
import_mep
[
'first_name'
],
last_name
=
import_mep
[
'last_name'
],
birthdate
=
import_mep
[
'birthdate'
],
defaults
=
updated_mep
)
inserted
+=
1
self
.
last_imported
=
inserted
self
.
save
()
setup.py
View file @
ff7abd46
...
...
@@ -9,6 +9,7 @@ requires = [
'django'
,
'djangorestframework'
,
'drf-nested-routers'
,
'requests'
,
'pillow'
,
]
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment