Commit b0cb38ec authored by jaster's avatar jaster

Add try/catch for all data import processes

All import processes now use a general try-catch during the import of each data
item. This ensures that even in the event of an error (due to a malformed data
piece for example), the import process will continue.

Errors caught this way are logged at the debug level.
parent 3abe72b1
......@@ -380,7 +380,11 @@ def main(stream=None):
for data in ijson.items(stream or sys.stdin, ''):
for rep in data:
try:
if rep['chambre'] == 'AN':
an_importer.manage_rep(rep)
elif rep['chambre'] == 'SEN':
sen_importer.manage_rep(rep)
except Exception as err:
logger.debug('error trying to import rep %s (%s)',
str(rep), str(err))
......@@ -420,7 +420,12 @@ def main(stream=None):
GenericImporter.pre_import(importer)
for data in ijson.items(stream or sys.stdin, 'item'):
try:
importer.manage_mep(data)
except Exception as err:
logger.debug('error trying to import rep %s (%s)',
str(data), str(err))
# Commenting for now, it's a bit dangerous, if a json file was corrupt it
# would drop valid data !
# importer.post_import()
......@@ -45,6 +45,7 @@ class PositionImporter:
return rep
def import_row(self, row):
try:
if len(row['date']) == 0:
if len(row['url']) == 0:
row['date'] = '2010-01-01'
......@@ -83,6 +84,10 @@ class PositionImporter:
row['first_name'], row['last_name'], row['url']))
return True
except Exception as err:
logger.debug('error trying to import position %s (%s)',
str(row), str(err))
return False
def main(stream=None):
......
......@@ -54,6 +54,7 @@ class RecommendationImporter:
return None
def import_row(self, row):
try:
dossier = self.get_dossier(row['title'])
if dossier is None:
logger.warn('No dossier "%s"' % row['title'])
......@@ -81,13 +82,18 @@ class RecommendationImporter:
weight=weight
)
recom.save()
logger.info('Created recommendation with weight %s for %s: %s' % (
logger.info('Created recommendation with weight %s for %s: %s'
% (
weight,
row['title'],
row['part']
))
return True
except Exception as err:
logger.debug('error trying to import recommendation %s (%s)',
str(row), str(err))
return False
def main(stream=None):
......
......@@ -134,4 +134,8 @@ def main(stream=None):
an = Chamber.objects.get(abbreviation='AN')
sen = Chamber.objects.get(abbreviation='SEN')
for data in ijson.items(stream or sys.stdin, 'item'):
try:
parse_dossier_data(data, an, sen)
except Exception as err:
logger.debug('error trying to import dossier %s (%s)',
str(data), str(err))
......@@ -106,4 +106,8 @@ def main(stream=None):
importer = ScrutinImporter()
for data in ijson.items(stream or sys.stdin, 'item'):
try:
importer.parse_scrutin_data(data)
except Exception as err:
logger.debug('error trying to import scrutin %s (%s)',
str(data), str(err))
......@@ -126,6 +126,10 @@ def main(stream=None):
importer = VotesImporter()
for data in ijson.items(stream or sys.stdin, 'item'):
try:
importer.parse_vote_data(data)
except Exception as err:
logger.debug('error trying to import vote %s (%s)',
str(data), str(err))
importer.update_totals()
......@@ -94,4 +94,8 @@ def main(stream=None):
ep = Chamber.objects.get(abbreviation='EP')
for data in ijson.items(stream or sys.stdin, 'item'):
try:
parse_dossier_data(data, ep)
except Exception as err:
logger.debug('error trying to import dossier %s (%s)',
str(data), str(err))
......@@ -255,4 +255,8 @@ def main(stream=None):
command.init_cache()
for vote_data in ijson.items(stream or sys.stdin, 'item'):
try:
command.parse_vote_data(vote_data)
except Exception as err:
logger.debug('error trying to import vote %s (%s)',
str(vote_data), str(err))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment