from forms import AddViolation, SearchViolation from django.views.generic import ListView, FormView, DetailView from django.views.generic.list import MultipleObjectMixin from django.http import HttpResponse, HttpResponseRedirect, Http404 from django.shortcuts import render_to_response, get_object_or_404, redirect, render from django.template import RequestContext, loader, Context from django.core.files import File from django.core.servers.basehttp import FileWrapper from django.core import serializers from django.conf import settings from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.core.exceptions import ObjectDoesNotExist from django.core.mail import send_mail from django.contrib import messages from django.contrib.auth.models import User from django.utils.translation import ugettext_lazy as _ from django.db.models import Count from haystack.generic_views import SearchView from haystack.query import SearchQuerySet from models import Violation, Attachment, Comment, Confirmation, COUNTRIES, STATUS, Operator, FeaturedCase from tempfile import mkstemp from datetime import datetime import hashlib, os, re, json from random import randint from email.mime.text import MIMEText from email.header import Header from urlparse import urljoin from BeautifulSoup import BeautifulSoup, Comment as BComment from operator import itemgetter from itertools import groupby def sanitizeHtml(value, base_url=None): rjs = r'[\s]*(&#x.{1,7})?'.join(list('javascript:')) rvb = r'[\s]*(&#x.{1,7})?'.join(list('vbscript:')) re_scripts = re.compile('(%s)|(%s)' % (rjs, rvb), re.IGNORECASE) validTags = 'p i strong b u a h1 h2 h3 pre br img'.split() validAttrs = 'href src width height'.split() urlAttrs = 'href src'.split() # Attributes which should have a URL soup = BeautifulSoup(value) for comment in soup.findAll(text=lambda text: isinstance(text, BComment)): # Get rid of comments comment.extract() for tag in soup.findAll(True): if tag.name not in validTags: tag.hidden = True attrs = tag.attrs tag.attrs = [] for attr, val in attrs: if attr in validAttrs: val = re_scripts.sub('', val) # Remove scripts (vbs & js) if attr in urlAttrs: val = urljoin(base_url, val) # Calculate the absolute url tag.attrs.append((attr, val)) return soup.renderContents().decode('utf8') def activate(request): try: v=Violation.objects.get(activationid=request.GET.get('key','asdf')) except: messages.add_message(request, messages.INFO, unicode(_('Thank you, this key has been already activated'))) return HttpResponseRedirect('/') # Redirect after POST if v: actid = hashlib.sha1(''.join([chr(randint(32, 122)) for x in range(12)])).hexdigest() to=[x.email for x in User.objects.filter(groups__name='moderator')] details='\n'.join(["%s: %s" % (k.capitalize(), val) for k,val in v.__dict__.items() if not k.startswith('_') and val]) msg = {'from': 'nnmon@respectmynet.eu', 'to': to, 'subject': 'NNMon submission approval'.encode("Utf-8"), 'body': "A new report was submitted. To approve click here: %s/moderate/?key=%s\n\nDetails follow:\n%s\n%s" % (settings.ROOT_URL or 'http://localhost:8001/', actid, details, v.comment_set.get().comment) } send_mail(msg['subject'], msg['body'], msg['from'], msg['to'], fail_silently=False) v.activationid=actid v.save() messages.add_message(request, messages.INFO, _('Thank you for verifying your submission. It will be listed shortly, after we\'ve checked that the report is valid.').encode("Utf-8")) return HttpResponseRedirect('/') # Redirect after POST def moderate(request): try: v=Violation.objects.get(activationid=request.GET.get('key','asdf')) except: messages.add_message(request, messages.INFO, unicode(_('Thank you, this key has been already activated'))) return HttpResponseRedirect('/') # Redirect after POST if not v: messages.add_message(request, messages.INFO, _('No such key')) return HttpResponseRedirect('/') # Redirect after POST if request.GET.get('action','')=='approve': messages.add_message(request, messages.INFO, _('Thank you for approving the submission.' % v.get_absolute_url())) msg = {'from': 'nnmon@respectmynet.eu', 'to': [v.comment_set.get().submitter_email], 'subject': _('NNMon submission approved').encode("Utf-8"), 'body': _("Your report has been approved.\nTo see it, please visit: %s/%s") % (settings.ROOT_URL or 'http://localhost:8001/', v.get_absolute_url()) } send_mail(msg['subject'], msg['body'], msg['from'], msg['to'], fail_silently=False) if settings.TWITTER_API: try: settings.TWITTER_API.PostUpdate("New #NetNeutrality violation reported for %s (%s) %s %s/%s" % (v.operator, v.country, v.contract, settings.ROOT_URL or 'http://localhost:8001/', v.id)) except: pass v.activationid='' v.save() return redirect(v) # Redirect after POST to violation url if request.GET.get('action','')=='delete': v.delete() messages.add_message(request, messages.INFO, _('Thank you for deleting the submission.')) return HttpResponseRedirect('/') # Redirect after POST return render_to_response('view.html', { 'v': v, 'key': request.GET.get('key') },context_instance=RequestContext(request)) def confirm(request, id, name=None): if name: if Confirmation.objects.filter(email=name, violation=id).count()==0: msg=_("Thank you for confirming a case. To finalize your confirmation please validate using your confirmation key.\nYour confirmation key is %s/%s%s") actid=sendverifymail('confirm/',name, msg) try: c=Confirmation(key=actid, email=name, violation=Violation.objects.get(pk=id)) except: return HttpResponse(unicode(_('Thank you, this has been already confirmed'))) c.save() return HttpResponse(unicode(_('Thank you for your confirmation'))) try: c = get_object_or_404(Confirmation, key=id) except: messages.add_message(request, messages.INFO, unicode(_("Thank you, this has been already confirmed"))) return HttpResponseRedirect('/') # Redirect after POST c.key='' c.save() messages.add_message(request, messages.INFO, unicode(_('Thank you for verifying your confirmation'))) return HttpResponseRedirect('/') # Redirect after POST def sendverifymail(service,to,body): actid = hashlib.sha1(''.join([chr(randint(32, 122)) for x in range(12)])).hexdigest() msg = {'from': 'nnmon@respectmynet.eu', 'to': [to.encode("Utf-8")], 'subject': _('NNMon submission verification').encode("Utf-8"), 'body': body % (settings.ROOT_URL or 'http://localhost:8001/', service, actid), } send_mail(msg['subject'], msg['body'], msg['from'], msg['to'], fail_silently=False) return actid class AddForm(FormView): template_name = 'index.html' form_class = AddViolation success_url = '/' def form_valid(self, form): #msg=_("Thank you for submitting a new report. To finalize your submission please confirm using your validation key.\nYour verification key is %s/%s%s\nPlease note that reports are moderated, it might take some time before your report appears online. Thank you for your patience.") #actid=sendverifymail('activate?key=',form.cleaned_data['email'], msg) actid = hashlib.sha1(''.join([chr(randint(32, 122)) for x in range(12)])).hexdigest() operator, created = Operator.objects.get_or_create(name=form.cleaned_data['operator']) v=Violation( country = form.cleaned_data['country'], operator_ref = operator, contract = form.cleaned_data['contract'], resource = form.cleaned_data['resource'], resource_name = form.cleaned_data['resource_name'], type = form.cleaned_data['type'], media = form.cleaned_data['media'], temporary = form.cleaned_data['temporary'], contractual = form.cleaned_data['contractual'], contract_excerpt = sanitizeHtml(form.cleaned_data['contract_excerpt']), loophole = form.cleaned_data['loophole'], activationid = actid, old = False ) v.save() conf=Confirmation(key=actid, email=form.cleaned_data['email'], violation=v) conf.save() c = Comment( comment=form.cleaned_data['comment'], submitter_email=form.cleaned_data['email'], submitter_name=form.cleaned_data['nick'], consent=form.cleaned_data['consent'], timestamp=datetime.now(), violation=v, ) c.save() for f in self.request.FILES.getlist('attachments[]'): a=Attachment(comment=c, name=f.name, type=f.content_type) m = hashlib.sha256() for chunk in f.chunks(): m.update(chunk) sname=m.hexdigest() a.storage.save(sname,f) a.save() messages.add_message(self.request, messages.INFO, _('Thank you for submitting this report, you will receive a verification email immediately, if not check your spam folder.')) return super(AddForm, self).form_valid(form) def get_context_data(self, **kwargs): context = super(AddForm, self).get_context_data(**kwargs) reports = sorted([(i['total'],i['id']) for i in Violation.objects.values('id').exclude(state__in=['closed', 'ooscope', 'duplicate']).exclude(old=True).annotate(total=Count('confirmation'))], reverse=True) confirms = sorted([(i['total'],i['country']) for i in Violation.objects.values('country').exclude(state__in=['closed', 'ooscope', 'duplicate']).exclude(old=True).annotate(total=Count('confirmation'))], reverse=True) operators = sorted([(i['total'],i['operator_ref__name']) for i in Violation.objects.values('operator_ref__name').exclude(state__in=['closed', 'ooscope', 'duplicate']).exclude(old=True).annotate(total=Count('confirmation'))], reverse=True) context['stats'] = [ (_('Total confirmed reports'), len([i for i,z in reports if i>0])), (_('Countries with some confirmed reports'), len([i for i,z in confirms if i>0])), (_('Operators with some confirmed reports'), len([i for i,z in operators if i>0]))] context['violations'] = [fc.case for fc in FeaturedCase.objects.all() if not fc.case.old][:3] return context class ViolationsList(ListView): template_name = 'list.html' context_object_name = 'violations' def get_queryset(self): queryset = Violation.objects.exclude(old=True) if 'operator' in self.kwargs: # If i Have operator I have a country queryset = Violation.objects.filter(country=self.kwargs['country'], operator_ref__name=self.kwargs['operator']) if 'country' in self.kwargs: queryset = Violation.objects.filter(country=self.kwargs['country']) if 'all' not in self.request.GET: queryset = queryset.exclude(state__in=['ooscope', 'duplicate', 'closed']) return queryset def get_context_data(self, **kwargs): context = super(ViolationsList, self).get_context_data(**kwargs) if 'country' in self.kwargs: context['country'] = self.kwargs['country'] else: countries = sorted([(i['total'], i['country']) for i in Violation.objects.values('country').exclude(state__in=['ooscope', 'duplicate', 'closed']).exclude(old=True).annotate(total=Count('country'))], reverse=True) countryweights=json.dumps([{'iso2': y, 'w': x} for x, y in countries]) context['countries'] = countries context['countryweights'] = countryweights return context class ViolationView(DetailView): model = Violation template_name = 'view.html' pk_url_kwarg = 'id' context_object_name = 'v' def get_object(self): object = super(ViolationView, self).get_object() if object.activationid: raise Http404 return object class LookupView(SearchView): searchqueryset = SearchQuerySet().exclude(old=True) form_class = SearchViolation def get_queryset(self): return super(LookupView, self).get_queryset().exclude(old=True) def get_context_data(self, *args, **kwargs): context = super(LookupView, self).get_context_data(*args, **kwargs) if 'object_list' in context: context['object_list'] = [obj for obj in context['object_list'] if not obj.object.old] return context class ViolationSearchView(SearchView): searchqueryset = SearchQuerySet().exclude(old=True) form_class = SearchViolation def get_queryset(self): return super(ViolationSearchView, self).get_queryset().exclude(old=True) def get_context_data(self, *args, **kwargs): context = super(ViolationSearchView, self).get_context_data(*args, **kwargs) countries = sorted([(k, len(list(g)),) for k, g in groupby(sorted([i['country'] for i in self.queryset.values('country')]))]) countryweights=json.dumps([{'iso2': x, 'w': y} for x, y in countries]) operators = sorted([(k, len(list(g)),) for k, g in groupby(sorted([i['operator_name'] for i in self.queryset.values('operator_name')]))]) context['operators'] = operators context['countries'] = countries context['countryweights'] = countryweights context['stats'] = [ (_('Number of reports in this search'), len(self.queryset.all())), (_('Countries with some confirmed reports'), len([c for c,v in countries if v > 0])), (_('Operators with some confirmed reports'), len([c for c,v in operators if v > 0])), ] return context def get_attach(request,id): f = get_object_or_404(Attachment, pk=id) wrapper = FileWrapper(f.storage) response=HttpResponse(wrapper, mimetype=f.type) response['Content-Disposition'] = 'attachment; filename="%s"' % f.name response['Content-Length'] = f.storage.size return response class ViolationCsvList(ListView): model = Violation template_name = 'csv.tmpl' content_type = 'text/csv' csv_name = 'respectmynet.csv' queryset = Violation.objects.all().annotate(confirmations=Count('confirmation')).order_by('-confirmations') def render_to_response(self, context, **kwargs): response = super(ListView, self).render_to_response(context, **kwargs) response['Content-Disposition']= 'attachment; filename={0}'.format(self.csv_name) return response def ascsv(request): response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename=respectmynet.csv' res=[] for v in Violation.objects.filter(activationid=''): res.append((v.state, v.country, v.operator, v.contract, v.resource, v.resource_name, v.type, v.media, v.temporary, v.contractual, v.contract_excerpt, v.loophole, v.editorial,v.comment_set.get().comment)) t = loader.get_template('csv.tmpl') c = Context({ 'data': res, }) response.write(t.render(c)) return response from sheet import save_ods def asods(request): response = HttpResponse(mimetype='application/vnd.oasis.opendocument.spreadsheet') response['Content-Disposition'] = 'attachment; filename=respectmynet-ec_berec_tm_questionnaire.ods' save_ods() f=open('/tmp/ec_berec_tm_questionnaire.ods','r') response.write(f.read()) f.close() return response