views.py

from forms import AddViolation, SearchViolation
from django.views.generic import ListView, FormView, DetailView
from django.views.generic.list import MultipleObjectMixin
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render_to_response, get_object_or_404, redirect
from django.template import RequestContext, loader, Context
from django.core.files import File
from django.core.servers.basehttp import FileWrapper
from django.core import serializers
from django.conf import settings
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.exceptions import ObjectDoesNotExist
from django.core.mail import send_mail
from django.contrib import messages
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from django.db.models import Count
from haystack.generic_views import SearchView
from haystack.query import SearchQuerySet
from models import Violation, Attachment, Comment, Confirmation, COUNTRIES, STATUS, Operator, FeaturedCase
from tempfile import mkstemp
from datetime import datetime
import hashlib, os, re, json
from random import randint
from email.mime.text import MIMEText
from email.header import Header
from urlparse import urljoin
from BeautifulSoup import BeautifulSoup, Comment as BComment
from operator import itemgetter
from itertools import groupby

def sanitizeHtml(value, base_url=None):
    rjs = r'[\s]*(&#x.{1,7})?'.join(list('javascript:'))
    rvb = r'[\s]*(&#x.{1,7})?'.join(list('vbscript:'))
    re_scripts = re.compile('(%s)|(%s)' % (rjs, rvb), re.IGNORECASE)
    validTags = 'p i strong b u a h1 h2 h3 pre br img'.split()
    validAttrs = 'href src width height'.split()
    urlAttrs = 'href src'.split() # Attributes which should have a URL
    soup = BeautifulSoup(value)
    for comment in soup.findAll(text=lambda text: isinstance(text, BComment)):
        # Get rid of comments
        comment.extract()
    for tag in soup.findAll(True):
        if tag.name not in validTags:
            tag.hidden = True
        attrs = tag.attrs
        tag.attrs = []
        for attr, val in attrs:
            if attr in validAttrs:
                val = re_scripts.sub('', val) # Remove scripts (vbs & js)
                if attr in urlAttrs:
                    val = urljoin(base_url, val) # Calculate the absolute url
                tag.attrs.append((attr, val))

    return soup.renderContents().decode('utf8')

def activate(request):
    try:
        v=Violation.objects.get(activationid=request.GET.get('key','asdf'))
    except:
        messages.add_message(request, messages.INFO, unicode(_('Thank you, this key has been already activated')))
        return HttpResponseRedirect('/') # Redirect after POST
    if v:
        actid = hashlib.sha1(''.join([chr(randint(32, 122)) for x in range(12)])).hexdigest()
        to=[x.email for x in User.objects.filter(groups__name='moderator')]
        details='\n'.join(["%s: %s" % (k.capitalize(), val) for k,val in v.__dict__.items() if not k.startswith('_') and val])

        msg = {'from': 'nnmon@respectmynet.eu',
               'to': to,
               'subject': 'NNMon submission approval'.encode("Utf-8"),
               'body': "A new report was submitted. To approve click here: %s/moderate/?key=%s\n\nDetails follow:\n%s\n%s" %
                       (settings.ROOT_URL or 'http://localhost:8001/', actid, details, v.comment_set.get().comment)
               }
        send_mail(msg['subject'], msg['body'], msg['from'], msg['to'], fail_silently=False)

        v.activationid=actid
        v.save()
        messages.add_message(request, messages.INFO, _('Thank you for verifying your submission. It will be listed shortly, after we\'ve checked that the report is valid.').encode("Utf-8"))
    return HttpResponseRedirect('/') # Redirect after POST

def moderate(request):
    try:
        v=Violation.objects.get(activationid=request.GET.get('key','asdf'))
    except:
        messages.add_message(request, messages.INFO, unicode(_('Thank you, this key has been already activated')))
        return HttpResponseRedirect('/') # Redirect after POST
    if not v:
        messages.add_message(request, messages.INFO, _('No such key'))
        return HttpResponseRedirect('/') # Redirect after POST
    if request.GET.get('action','')=='approve':
        messages.add_message(request, messages.INFO, _('Thank you for approving the <a href="%s">submission</a>.' % v.get_absolute_url()))

        msg = {'from': 'nnmon@respectmynet.eu',
               'to': [v.comment_set.get().submitter_email],
               'subject': _('NNMon submission approved').encode("Utf-8"),
               'body': _("Your report has been approved.\nTo see it, please visit: %s/%s") %
                      (settings.ROOT_URL or 'http://localhost:8001/', v.get_absolute_url())
               }
        send_mail(msg['subject'], msg['body'], msg['from'], msg['to'], fail_silently=False)

        if settings.TWITTER_API:
            try:
                settings.TWITTER_API.PostUpdate("New #NetNeutrality violation reported for %s (%s) %s %s/%s" % (v.operator, v.country, v.contract, settings.ROOT_URL or 'http://localhost:8001/', v.id))
            except:
                pass
        v.activationid=''
        v.save()
        return redirect(v) # Redirect after POST to violation url
    if request.GET.get('action','')=='delete':
        v.delete()
        messages.add_message(request, messages.INFO, _('Thank you for deleting the submission.'))
        return HttpResponseRedirect('/') # Redirect after POST
    return render_to_response('view.html', { 'v': v, 'key': request.GET.get('key') },context_instance=RequestContext(request))

def confirm(request, id, name=None):
    if name:
        if Confirmation.objects.filter(email=name, violation=id).count()==0:
            msg=_("Thank you for confirming a case. To finalize your confirmation please validate using your confirmation key.\nYour confirmation key is %s/%s%s")
            actid=sendverifymail('confirm/',name, msg)
            try:
                c=Confirmation(key=actid, email=name, violation=Violation.objects.get(pk=id))
            except:
                return HttpResponse(unicode(_('Thank you, this has been already confirmed')))
            c.save()
        return HttpResponse(unicode(_('Thank you for your confirmation')))
    try:
        c = get_object_or_404(Confirmation, key=id)
    except:
        messages.add_message(request, messages.INFO, unicode(_("Thank you, this has been already confirmed")))
        return HttpResponseRedirect('/') # Redirect after POST
    c.key=''
    c.save()
    messages.add_message(request, messages.INFO, unicode(_('Thank you for verifying your confirmation')))
    return HttpResponseRedirect('/') # Redirect after POST

def sendverifymail(service,to,body):
    actid = hashlib.sha1(''.join([chr(randint(32, 122)) for x in range(12)])).hexdigest()
    msg = {'from': 'nnmon@respectmynet.eu',
           'to': [to.encode("Utf-8")],
           'subject': _('NNMon submission verification').encode("Utf-8"),
           'body': body % (settings.ROOT_URL or 'http://localhost:8001/', service, actid),
           }
    send_mail(msg['subject'], msg['body'], msg['from'], msg['to'], fail_silently=False)

    return actid

class AddForm(FormView):
    template_name = 'index.html'
    form_class = AddViolation
    success_url = '/'

    def form_valid(self, form):
        msg=_("Thank you for submitting a new report. To finalize your submission please confirm using your validation key.\nYour verification key is %s/%s%s\nPlease note that reports are moderated, it might take some time before your report appears online. Thank you for your patience.")
        actid=sendverifymail('activate?key=',form.cleaned_data['email'], msg)
        operator, created = Operator.objects.get_or_create(name=form.cleaned_data['operator'])
        v=Violation(
                country = form.cleaned_data['country'],
                operator_ref = operator,
                contract = form.cleaned_data['contract'],
                resource = form.cleaned_data['resource'],
                resource_name = form.cleaned_data['resource_name'],
                type = form.cleaned_data['type'],
                media = form.cleaned_data['media'],
                temporary = form.cleaned_data['temporary'],
                contractual = form.cleaned_data['contractual'],
                contract_excerpt = sanitizeHtml(form.cleaned_data['contract_excerpt']),
                loophole = form.cleaned_data['loophole'],
                activationid = actid,
                old = False
                )
        v.save()
        #c=Confirmation(key='', email=form.cleaned_data['email'], violation=v)
        #c.save()
        c = Comment(
            comment=form.cleaned_data['comment'],
            submitter_email=form.cleaned_data['email'],
            submitter_name=form.cleaned_data['nick'],
            consent=form.cleaned_data['consent'],
            timestamp=datetime.now(),
            violation=v,
            )
        c.save()
        for f in self.request.FILES.getlist('attachments[]'):
            a=Attachment(comment=c, name=f.name, type=f.content_type)
            m = hashlib.sha256()
            for chunk in f.chunks():
                m.update(chunk)
            sname=m.hexdigest()
            a.storage.save(sname,f)
            a.save()
        messages.add_message(self.request, messages.INFO, _('Thank you for submitting this report, you will receive a verification email immediately, if not check your spam folder.'))
        return super(AddForm, self).form_valid(form)

    def get_context_data(self, **kwargs):
        context = super(AddForm, self).get_context_data(**kwargs)
        reports = sorted([(i['total'],i['id'])
                         for i in Violation.objects.values('id').filter(activationid='').exclude(state__in=['closed', 'ooscope', 'duplicate']).exclude(old=True).annotate(total=Count('confirmation'))],
                    reverse=True)
        confirms = sorted([(i['total'],i['country'])
                         for i in Violation.objects.values('country').filter(activationid='').exclude(state__in=['closed', 'ooscope', 'duplicate']).exclude(old=True).annotate(total=Count('confirmation'))],
                    reverse=True)
        operators = sorted([(i['total'],i['operator_ref__name'])
                         for i in Violation.objects.values('operator_ref__name').filter(activationid='').exclude(state__in=['closed', 'ooscope', 'duplicate']).exclude(old=True).annotate(total=Count('confirmation'))],
                     reverse=True)
        context['stats'] = [
                (_('Total confirmed reports'), len([i for i,z in reports if i>0])),
                (_('Countries with some confirmed reports'), len([i for i,z in confirms if i>0])),
                (_('Operators with some confirmed reports'), len([i for i,z in operators if i>0]))]
        context['violations'] = [fc.case for fc in FeaturedCase.objects.all() if not fc.case.old][:3]
        return context

class ViolationsList(ListView):
    template_name = 'list.html'
    context_object_name = 'violations'

    def get_queryset(self):
        queryset = Violation.objects.filter(activationid='').exclude(old=True)
        if 'operator' in self.kwargs:
            # If i Have operator I have a country
            queryset = Violation.objects.filter(activationid='',
                    country=self.kwargs['country'],
                    operator_ref__name=self.kwargs['operator']).exclude(old=True)
        if 'country' in self.kwargs:
            queryset = Violation.objects.filter(activationid='',
                    country=self.kwargs['country']).exclude(old=True)
        if 'all' not in self.request.GET:
            queryset = queryset.filter(activationid='').exclude(state__in=['duplicate', 'closed']).exclude(old=False)
        return queryset

    def get_context_data(self, **kwargs):
        context = super(ViolationsList, self).get_context_data(**kwargs)

        if 'country' in self.kwargs:
            context['country'] = self.kwargs['country']
        else:
            countries = sorted([(i['total'], i['country'])
                for i in Violation.objects.values('country').filter(activationid='').exclude(state__in=['duplicate', 'closed']).exclude(old=True).annotate(total=Count('country'))],
                reverse=True)
            countryweights=json.dumps([{'iso2': y, 'w': x} for x, y in countries])
            context['countries'] = countries
            context['countryweights'] = countryweights
        return context

class ViolationView(DetailView):
    model = Violation
    template_name = 'view.html'
    pk_url_kwarg = 'id'
    context_object_name = 'v'

    def get_object(self):
        object = super(ViolationView, self).get_object()
        if object.activationid:
            raise Http404
        return object

class LookupView(SearchView):
    searchqueryset = SearchQuerySet().exclude(old=True)
    form_class = SearchViolation

    def get_queryset(self):
        return super(LookupView, self).get_queryset().exclude(old=True)

    def get_context_data(self, *args, **kwargs):
        context = super(LookupView, self).get_context_data(*args, **kwargs)
        if 'object_list' in context:
            context['object_list'] = [obj for obj in context['object_list'] if not obj.object.old]
        return context

class ViolationSearchView(SearchView):
    searchqueryset = SearchQuerySet().exclude(old=True)
    form_class = SearchViolation

    def get_queryset(self):
        return super(ViolationSearchView, self).get_queryset().exclude(old=True)

    def get_context_data(self, *args, **kwargs):
        context = super(ViolationSearchView, self).get_context_data(*args, **kwargs)
        countries = sorted([(k, len(list(g)),) for k, g in groupby(sorted([i['country'] for i in self.queryset.values('country')]))])
        countryweights=json.dumps([{'iso2': x, 'w': y} for x, y in countries])
        operators = sorted([(k, len(list(g)),) for k, g in groupby(sorted([i['operator_name'] for i in self.queryset.values('operator_name')]))])
        context['operators'] = operators
        context['countries'] = countries
        context['countryweights'] = countryweights
        context['stats'] = [
                (_('Number of reports in this search'), len(self.queryset.all())),
                (_('Countries with some confirmed reports'), len([c for c,v in countries if v > 0])),
                (_('Operators with some confirmed reports'), len([c for c,v in operators if v > 0])),
                ]
        return context

def get_attach(request,id):
    f = get_object_or_404(Attachment, pk=id)
    wrapper = FileWrapper(f.storage)
    response=HttpResponse(wrapper, mimetype=f.type)
    response['Content-Disposition'] = 'attachment; filename="%s"' % f.name
    response['Content-Length'] = f.storage.size
    return response

def ascsv(request):
    response = HttpResponse(mimetype='text/csv')
    response['Content-Disposition'] = 'attachment; filename=respectmynet.csv'

    res=[]
    for v in Violation.objects.filter(activationid=''):
        res.append((v.state, v.country, v.operator, v.contract, v.resource, v.resource_name, v.type, v.media, v.temporary, v.contractual, v.contract_excerpt, v.loophole, v.editorial,v.comment_set.get().comment))
    t = loader.get_template('csv.tmpl')
    c = Context({
        'data': res,
    })
    response.write(t.render(c))
    return response

from sheet import save_ods
def asods(request):
    response = HttpResponse(mimetype='application/vnd.oasis.opendocument.spreadsheet')
    response['Content-Disposition'] = 'attachment; filename=respectmynet-ec_berec_tm_questionnaire.ods'
    save_ods()
    f=open('/tmp/ec_berec_tm_questionnaire.ods','r')
    response.write(f.read())
    f.close()
    return response