Commit b0cb38ec authored by jaster's avatar jaster
Browse files

Add try/catch for all data import processes

All import processes now use a general try-catch during the import of each data
item. This ensures that even in the event of an error (due to a malformed data
piece for example), the import process will continue.

Errors caught this way are logged at the debug level.
parent 3abe72b1
...@@ -380,7 +380,11 @@ def main(stream=None): ...@@ -380,7 +380,11 @@ def main(stream=None):
for data in ijson.items(stream or sys.stdin, ''): for data in ijson.items(stream or sys.stdin, ''):
for rep in data: for rep in data:
try:
if rep['chambre'] == 'AN': if rep['chambre'] == 'AN':
an_importer.manage_rep(rep) an_importer.manage_rep(rep)
elif rep['chambre'] == 'SEN': elif rep['chambre'] == 'SEN':
sen_importer.manage_rep(rep) sen_importer.manage_rep(rep)
except Exception as err:
logger.debug('error trying to import rep %s (%s)',
str(rep), str(err))
...@@ -420,7 +420,12 @@ def main(stream=None): ...@@ -420,7 +420,12 @@ def main(stream=None):
GenericImporter.pre_import(importer) GenericImporter.pre_import(importer)
for data in ijson.items(stream or sys.stdin, 'item'): for data in ijson.items(stream or sys.stdin, 'item'):
try:
importer.manage_mep(data) importer.manage_mep(data)
except Exception as err:
logger.debug('error trying to import rep %s (%s)',
str(data), str(err))
# Commenting for now, it's a bit dangerous, if a json file was corrupt it # Commenting for now, it's a bit dangerous, if a json file was corrupt it
# would drop valid data ! # would drop valid data !
# importer.post_import() # importer.post_import()
...@@ -45,6 +45,7 @@ class PositionImporter: ...@@ -45,6 +45,7 @@ class PositionImporter:
return rep return rep
def import_row(self, row): def import_row(self, row):
try:
if len(row['date']) == 0: if len(row['date']) == 0:
if len(row['url']) == 0: if len(row['url']) == 0:
row['date'] = '2010-01-01' row['date'] = '2010-01-01'
...@@ -83,6 +84,10 @@ class PositionImporter: ...@@ -83,6 +84,10 @@ class PositionImporter:
row['first_name'], row['last_name'], row['url'])) row['first_name'], row['last_name'], row['url']))
return True return True
except Exception as err:
logger.debug('error trying to import position %s (%s)',
str(row), str(err))
return False
def main(stream=None): def main(stream=None):
......
...@@ -54,6 +54,7 @@ class RecommendationImporter: ...@@ -54,6 +54,7 @@ class RecommendationImporter:
return None return None
def import_row(self, row): def import_row(self, row):
try:
dossier = self.get_dossier(row['title']) dossier = self.get_dossier(row['title'])
if dossier is None: if dossier is None:
logger.warn('No dossier "%s"' % row['title']) logger.warn('No dossier "%s"' % row['title'])
...@@ -81,13 +82,18 @@ class RecommendationImporter: ...@@ -81,13 +82,18 @@ class RecommendationImporter:
weight=weight weight=weight
) )
recom.save() recom.save()
logger.info('Created recommendation with weight %s for %s: %s' % ( logger.info('Created recommendation with weight %s for %s: %s'
% (
weight, weight,
row['title'], row['title'],
row['part'] row['part']
)) ))
return True return True
except Exception as err:
logger.debug('error trying to import recommendation %s (%s)',
str(row), str(err))
return False
def main(stream=None): def main(stream=None):
......
...@@ -134,4 +134,8 @@ def main(stream=None): ...@@ -134,4 +134,8 @@ def main(stream=None):
an = Chamber.objects.get(abbreviation='AN') an = Chamber.objects.get(abbreviation='AN')
sen = Chamber.objects.get(abbreviation='SEN') sen = Chamber.objects.get(abbreviation='SEN')
for data in ijson.items(stream or sys.stdin, 'item'): for data in ijson.items(stream or sys.stdin, 'item'):
try:
parse_dossier_data(data, an, sen) parse_dossier_data(data, an, sen)
except Exception as err:
logger.debug('error trying to import dossier %s (%s)',
str(data), str(err))
...@@ -106,4 +106,8 @@ def main(stream=None): ...@@ -106,4 +106,8 @@ def main(stream=None):
importer = ScrutinImporter() importer = ScrutinImporter()
for data in ijson.items(stream or sys.stdin, 'item'): for data in ijson.items(stream or sys.stdin, 'item'):
try:
importer.parse_scrutin_data(data) importer.parse_scrutin_data(data)
except Exception as err:
logger.debug('error trying to import scrutin %s (%s)',
str(data), str(err))
...@@ -126,6 +126,10 @@ def main(stream=None): ...@@ -126,6 +126,10 @@ def main(stream=None):
importer = VotesImporter() importer = VotesImporter()
for data in ijson.items(stream or sys.stdin, 'item'): for data in ijson.items(stream or sys.stdin, 'item'):
try:
importer.parse_vote_data(data) importer.parse_vote_data(data)
except Exception as err:
logger.debug('error trying to import vote %s (%s)',
str(data), str(err))
importer.update_totals() importer.update_totals()
...@@ -94,4 +94,8 @@ def main(stream=None): ...@@ -94,4 +94,8 @@ def main(stream=None):
ep = Chamber.objects.get(abbreviation='EP') ep = Chamber.objects.get(abbreviation='EP')
for data in ijson.items(stream or sys.stdin, 'item'): for data in ijson.items(stream or sys.stdin, 'item'):
try:
parse_dossier_data(data, ep) parse_dossier_data(data, ep)
except Exception as err:
logger.debug('error trying to import dossier %s (%s)',
str(data), str(err))
...@@ -255,4 +255,8 @@ def main(stream=None): ...@@ -255,4 +255,8 @@ def main(stream=None):
command.init_cache() command.init_cache()
for vote_data in ijson.items(stream or sys.stdin, 'item'): for vote_data in ijson.items(stream or sys.stdin, 'item'):
try:
command.parse_vote_data(vote_data) command.parse_vote_data(vote_data)
except Exception as err:
logger.debug('error trying to import vote %s (%s)',
str(vote_data), str(err))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment