diff --git a/src/representatives/contrib/francedata/import_representatives.py b/src/representatives/contrib/francedata/import_representatives.py index f88307fb5f60abf4148d84949ca20142a1b5ad69..2e8eb367157b35ac7b8d8c152dae2201d3a5eb1a 100644 --- a/src/representatives/contrib/francedata/import_representatives.py +++ b/src/representatives/contrib/francedata/import_representatives.py @@ -380,7 +380,10 @@ def main(stream=None): for data in ijson.items(stream or sys.stdin, ''): for rep in data: - if rep['chambre'] == 'AN': - an_importer.manage_rep(rep) - elif rep['chambre'] == 'SEN': - sen_importer.manage_rep(rep) + try: + if rep['chambre'] == 'AN': + an_importer.manage_rep(rep) + elif rep['chambre'] == 'SEN': + sen_importer.manage_rep(rep) + except Exception: + logger.exception('error trying to import rep %s', str(rep)) diff --git a/src/representatives/contrib/parltrack/import_representatives.py b/src/representatives/contrib/parltrack/import_representatives.py index c43f0493f77d0c91e402612daa2f52a0cae99b31..bd8c9f89833a88c4c7654880f8eeda577a5cb9aa 100644 --- a/src/representatives/contrib/parltrack/import_representatives.py +++ b/src/representatives/contrib/parltrack/import_representatives.py @@ -420,7 +420,11 @@ def main(stream=None): GenericImporter.pre_import(importer) for data in ijson.items(stream or sys.stdin, 'item'): - importer.manage_mep(data) + try: + importer.manage_mep(data) + except Exception: + logger.exception('error trying to import rep %s', str(data)) + # Commenting for now, it's a bit dangerous, if a json file was corrupt it # would drop valid data ! # importer.post_import() diff --git a/src/representatives_positions/contrib/import_positions.py b/src/representatives_positions/contrib/import_positions.py index ea233ea2358dc3d20fb969b3c3adc32c887c9da9..a64a0374abcbda66ecbf166a716cea3fe3d13b8b 100644 --- a/src/representatives_positions/contrib/import_positions.py +++ b/src/representatives_positions/contrib/import_positions.py @@ -45,44 +45,48 @@ class PositionImporter: return rep def import_row(self, row): - if len(row['date']) == 0: - if len(row['url']) == 0: - row['date'] = '2010-01-01' - row['url'] = '/' - else: - row['date'] = position_dates.get(row['url'], None) - - if row['date'] is None: - logger.warn('Dateless position for %s %s on URL %s' % - (row['first_name'], row['last_name'], row['url'])) - return False - - rep = self.get_rep(row['first_name'], row['last_name']) - if rep is None: - logger.warn('Could not find rep %s %s' % (row['first_name'], - row['last_name'])) - return False - - text = re.sub('(^

|

$)', '', row['content']) - if row['title'] is not None and len(row['title']) > 0: - text = '%s\n%s' % (row['title'], text) - try: - position = Position.objects.get(representative=rep, - link=row['url']) - except Position.DoesNotExist: - position = Position( - representative=rep, - link=row['url'], - datetime=row['date'], - text=text, - published=True - ) - position.save() - logger.info('Created position for %s %s on URL %s' % ( - row['first_name'], row['last_name'], row['url'])) - - return True + if len(row['date']) == 0: + if len(row['url']) == 0: + row['date'] = '2010-01-01' + row['url'] = '/' + else: + row['date'] = position_dates.get(row['url'], None) + + if row['date'] is None: + logger.warn('Dateless position for %s %s on URL %s' % + (row['first_name'], row['last_name'], row['url'])) + return False + + rep = self.get_rep(row['first_name'], row['last_name']) + if rep is None: + logger.warn('Could not find rep %s %s' % (row['first_name'], + row['last_name'])) + return False + + text = re.sub('(^

|

$)', '', row['content']) + if row['title'] is not None and len(row['title']) > 0: + text = '%s\n%s' % (row['title'], text) + + try: + position = Position.objects.get(representative=rep, + link=row['url']) + except Position.DoesNotExist: + position = Position( + representative=rep, + link=row['url'], + datetime=row['date'], + text=text, + published=True + ) + position.save() + logger.info('Created position for %s %s on URL %s' % ( + row['first_name'], row['last_name'], row['url'])) + + return True + except Exception: + logger.exception('error trying to import position %s', str(row)) + return False def main(stream=None): diff --git a/src/representatives_recommendations/contrib/import_recommendations.py b/src/representatives_recommendations/contrib/import_recommendations.py index c056afdd8d3f9191594e93b72d40a945cfe59d9a..2ee99034c2c33fabbd891a8483fd44f1849ef655 100644 --- a/src/representatives_recommendations/contrib/import_recommendations.py +++ b/src/representatives_recommendations/contrib/import_recommendations.py @@ -54,41 +54,47 @@ class RecommendationImporter: return None def import_row(self, row): - dossier = self.get_dossier(row['title']) - if dossier is None: - logger.warn('No dossier "%s"' % row['title']) - return False + try: + dossier = self.get_dossier(row['title']) + if dossier is None: + logger.warn('No dossier "%s"' % row['title']) + return False + + proposal = self.get_proposal(dossier, row['part']) + if proposal is None: + logger.warn('No proposal "%s" for dossier %s (%d): "%s"' % ( + row['part'].decode('utf-8'), dossier.reference, dossier.pk, + row['title'])) + return False + + weight = int(row['weight']) * int(row['ponderation']) + descr = row['description'].strip() + if len(descr) == 0: + descr = '%s on %s' % (row['part'], dossier.reference) - proposal = self.get_proposal(dossier, row['part']) - if proposal is None: - logger.warn('No proposal "%s" for dossier %s (%d): "%s"' % ( - row['part'].decode('utf-8'), dossier.reference, dossier.pk, - row['title'])) + try: + recom = Recommendation.objects.get(proposal=proposal) + except Recommendation.DoesNotExist: + recom = Recommendation( + proposal=proposal, + recommendation=row['recommendation'], + title=descr, + weight=weight + ) + recom.save() + logger.info('Created recommendation with weight %s for %s: %s' + % ( + weight, + row['title'], + row['part'] + )) + + return True + except Exception: + logger.exception('error trying to import recommendation %s', + str(row)) return False - weight = int(row['weight']) * int(row['ponderation']) - descr = row['description'].strip() - if len(descr) == 0: - descr = '%s on %s' % (row['part'], dossier.reference) - - try: - recom = Recommendation.objects.get(proposal=proposal) - except Recommendation.DoesNotExist: - recom = Recommendation( - proposal=proposal, - recommendation=row['recommendation'], - title=descr, - weight=weight - ) - recom.save() - logger.info('Created recommendation with weight %s for %s: %s' % ( - weight, - row['title'], - row['part'] - )) - - return True - def main(stream=None): """ diff --git a/src/representatives_votes/contrib/francedata/import_dossiers.py b/src/representatives_votes/contrib/francedata/import_dossiers.py index 5e5f640b522bc957466a75bec3087b18b0fe976d..3ad6fc764c2db540ae885c3e5b8c1f4260ada09b 100644 --- a/src/representatives_votes/contrib/francedata/import_dossiers.py +++ b/src/representatives_votes/contrib/francedata/import_dossiers.py @@ -134,4 +134,7 @@ def main(stream=None): an = Chamber.objects.get(abbreviation='AN') sen = Chamber.objects.get(abbreviation='SEN') for data in ijson.items(stream or sys.stdin, 'item'): - parse_dossier_data(data, an, sen) + try: + parse_dossier_data(data, an, sen) + except Exception: + logger.exception('error trying to import dossier %s', str(data)) diff --git a/src/representatives_votes/contrib/francedata/import_scrutins.py b/src/representatives_votes/contrib/francedata/import_scrutins.py old mode 100644 new mode 100755 index 3ce820843a93763fff2743f43c8c4ca4fa9fa3cf..4c483ed52f103172163c1c27fc8921093dea5c79 --- a/src/representatives_votes/contrib/francedata/import_scrutins.py +++ b/src/representatives_votes/contrib/francedata/import_scrutins.py @@ -106,4 +106,7 @@ def main(stream=None): importer = ScrutinImporter() for data in ijson.items(stream or sys.stdin, 'item'): - importer.parse_scrutin_data(data) + try: + importer.parse_scrutin_data(data) + except Exception: + logger.exception('error trying to import scrutin %s', str(data)) diff --git a/src/representatives_votes/contrib/francedata/import_votes.py b/src/representatives_votes/contrib/francedata/import_votes.py old mode 100644 new mode 100755 index 14125c00d09bd641c283e251e7f82ca6e0f6c5c5..114b35873f4bdec72992d01b0a6d82664f136cfe --- a/src/representatives_votes/contrib/francedata/import_votes.py +++ b/src/representatives_votes/contrib/francedata/import_votes.py @@ -126,6 +126,9 @@ def main(stream=None): importer = VotesImporter() for data in ijson.items(stream or sys.stdin, 'item'): - importer.parse_vote_data(data) + try: + importer.parse_vote_data(data) + except Exception: + logger.exception('error trying to import vote %s', str(data)) importer.update_totals() diff --git a/src/representatives_votes/contrib/parltrack/import_dossiers.py b/src/representatives_votes/contrib/parltrack/import_dossiers.py old mode 100644 new mode 100755 index 3728674256bc833ec9ba9a251df9122b03fcfeab..a84a35d63dfba2c12d37e86bb001b366cca3fc18 --- a/src/representatives_votes/contrib/parltrack/import_dossiers.py +++ b/src/representatives_votes/contrib/parltrack/import_dossiers.py @@ -94,4 +94,7 @@ def main(stream=None): ep = Chamber.objects.get(abbreviation='EP') for data in ijson.items(stream or sys.stdin, 'item'): - parse_dossier_data(data, ep) + try: + parse_dossier_data(data, ep) + except Exception: + logger.exception('error trying to import dossier %s', str(data)) diff --git a/src/representatives_votes/contrib/parltrack/import_votes.py b/src/representatives_votes/contrib/parltrack/import_votes.py old mode 100644 new mode 100755 index d0351e82bb1326062fbf571bce6fde01664752ec..2a91049a895c20d7290233fd94b96513bbfe9ff6 --- a/src/representatives_votes/contrib/parltrack/import_votes.py +++ b/src/representatives_votes/contrib/parltrack/import_votes.py @@ -255,4 +255,7 @@ def main(stream=None): command.init_cache() for vote_data in ijson.items(stream or sys.stdin, 'item'): - command.parse_vote_data(vote_data) + try: + command.parse_vote_data(vote_data) + except Exception: + logger.exception('error trying to import vote %s', str(vote_data))