Commit e293e3b9 authored by Jamesie Pic's avatar Jamesie Pic

Merge branch 'master' into 'master'

Add try/catch for all data import processes

See merge request !186
parents 3d475621 89729f5f
Pipeline #917 passed with stages
in 10 minutes and 28 seconds
...@@ -380,7 +380,10 @@ def main(stream=None): ...@@ -380,7 +380,10 @@ def main(stream=None):
for data in ijson.items(stream or sys.stdin, ''): for data in ijson.items(stream or sys.stdin, ''):
for rep in data: for rep in data:
try:
if rep['chambre'] == 'AN': if rep['chambre'] == 'AN':
an_importer.manage_rep(rep) an_importer.manage_rep(rep)
elif rep['chambre'] == 'SEN': elif rep['chambre'] == 'SEN':
sen_importer.manage_rep(rep) sen_importer.manage_rep(rep)
except Exception:
logger.exception('error trying to import rep %s', str(rep))
...@@ -420,7 +420,11 @@ def main(stream=None): ...@@ -420,7 +420,11 @@ def main(stream=None):
GenericImporter.pre_import(importer) GenericImporter.pre_import(importer)
for data in ijson.items(stream or sys.stdin, 'item'): for data in ijson.items(stream or sys.stdin, 'item'):
try:
importer.manage_mep(data) importer.manage_mep(data)
except Exception:
logger.exception('error trying to import rep %s', str(data))
# Commenting for now, it's a bit dangerous, if a json file was corrupt it # Commenting for now, it's a bit dangerous, if a json file was corrupt it
# would drop valid data ! # would drop valid data !
# importer.post_import() # importer.post_import()
...@@ -45,6 +45,7 @@ class PositionImporter: ...@@ -45,6 +45,7 @@ class PositionImporter:
return rep return rep
def import_row(self, row): def import_row(self, row):
try:
if len(row['date']) == 0: if len(row['date']) == 0:
if len(row['url']) == 0: if len(row['url']) == 0:
row['date'] = '2010-01-01' row['date'] = '2010-01-01'
...@@ -83,6 +84,9 @@ class PositionImporter: ...@@ -83,6 +84,9 @@ class PositionImporter:
row['first_name'], row['last_name'], row['url'])) row['first_name'], row['last_name'], row['url']))
return True return True
except Exception:
logger.exception('error trying to import position %s', str(row))
return False
def main(stream=None): def main(stream=None):
......
...@@ -54,6 +54,7 @@ class RecommendationImporter: ...@@ -54,6 +54,7 @@ class RecommendationImporter:
return None return None
def import_row(self, row): def import_row(self, row):
try:
dossier = self.get_dossier(row['title']) dossier = self.get_dossier(row['title'])
if dossier is None: if dossier is None:
logger.warn('No dossier "%s"' % row['title']) logger.warn('No dossier "%s"' % row['title'])
...@@ -81,13 +82,18 @@ class RecommendationImporter: ...@@ -81,13 +82,18 @@ class RecommendationImporter:
weight=weight weight=weight
) )
recom.save() recom.save()
logger.info('Created recommendation with weight %s for %s: %s' % ( logger.info('Created recommendation with weight %s for %s: %s'
% (
weight, weight,
row['title'], row['title'],
row['part'] row['part']
)) ))
return True return True
except Exception:
logger.exception('error trying to import recommendation %s',
str(row))
return False
def main(stream=None): def main(stream=None):
......
...@@ -134,4 +134,7 @@ def main(stream=None): ...@@ -134,4 +134,7 @@ def main(stream=None):
an = Chamber.objects.get(abbreviation='AN') an = Chamber.objects.get(abbreviation='AN')
sen = Chamber.objects.get(abbreviation='SEN') sen = Chamber.objects.get(abbreviation='SEN')
for data in ijson.items(stream or sys.stdin, 'item'): for data in ijson.items(stream or sys.stdin, 'item'):
try:
parse_dossier_data(data, an, sen) parse_dossier_data(data, an, sen)
except Exception:
logger.exception('error trying to import dossier %s', str(data))
...@@ -106,4 +106,7 @@ def main(stream=None): ...@@ -106,4 +106,7 @@ def main(stream=None):
importer = ScrutinImporter() importer = ScrutinImporter()
for data in ijson.items(stream or sys.stdin, 'item'): for data in ijson.items(stream or sys.stdin, 'item'):
try:
importer.parse_scrutin_data(data) importer.parse_scrutin_data(data)
except Exception:
logger.exception('error trying to import scrutin %s', str(data))
...@@ -126,6 +126,9 @@ def main(stream=None): ...@@ -126,6 +126,9 @@ def main(stream=None):
importer = VotesImporter() importer = VotesImporter()
for data in ijson.items(stream or sys.stdin, 'item'): for data in ijson.items(stream or sys.stdin, 'item'):
try:
importer.parse_vote_data(data) importer.parse_vote_data(data)
except Exception:
logger.exception('error trying to import vote %s', str(data))
importer.update_totals() importer.update_totals()
...@@ -94,4 +94,7 @@ def main(stream=None): ...@@ -94,4 +94,7 @@ def main(stream=None):
ep = Chamber.objects.get(abbreviation='EP') ep = Chamber.objects.get(abbreviation='EP')
for data in ijson.items(stream or sys.stdin, 'item'): for data in ijson.items(stream or sys.stdin, 'item'):
try:
parse_dossier_data(data, ep) parse_dossier_data(data, ep)
except Exception:
logger.exception('error trying to import dossier %s', str(data))
...@@ -255,4 +255,7 @@ def main(stream=None): ...@@ -255,4 +255,7 @@ def main(stream=None):
command.init_cache() command.init_cache()
for vote_data in ijson.items(stream or sys.stdin, 'item'): for vote_data in ijson.items(stream or sys.stdin, 'item'):
try:
command.parse_vote_data(vote_data) command.parse_vote_data(vote_data)
except Exception:
logger.exception('error trying to import vote %s', str(vote_data))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment