Commit a57380f5 authored by Okhin's avatar Okhin

Need to encode strings put in JSON

parent 42068cec
Pipeline #1176 passed with stage
in 7 seconds
...@@ -28,7 +28,7 @@ class FrancedataPipeline(object): ...@@ -28,7 +28,7 @@ class FrancedataPipeline(object):
self.json = gzip.open(outfile, 'wb') self.json = gzip.open(outfile, 'wb')
def spider_opened(self, spider): def spider_opened(self, spider):
self.json.write('[') self.json.write('['.encode())
try: try:
spider.set_pipeline(self) spider.set_pipeline(self)
...@@ -43,7 +43,7 @@ class FrancedataPipeline(object): ...@@ -43,7 +43,7 @@ class FrancedataPipeline(object):
self.urls.add(item['url']) self.urls.add(item['url'])
if self.has_items: if self.has_items:
self.json.write(',\n') self.json.write(',\n'.encode())
json.dump(item, self.json, cls=ScrapyJSONEncoder) json.dump(item, self.json, cls=ScrapyJSONEncoder)
self.has_items = True self.has_items = True
...@@ -51,5 +51,5 @@ class FrancedataPipeline(object): ...@@ -51,5 +51,5 @@ class FrancedataPipeline(object):
return item return item
def spider_closed(self, spider): def spider_closed(self, spider):
self.json.write(']') self.json.write(']'.encode())
self.json.close() self.json.close()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment