models.py

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core import files
from django.core.files.base import ContentFile

from taggit.managers import TaggableManager
from newspaper import Article as ArticleParser
from django_und.models import VoteMixin
from django_fsm import FSMField, transition, RETURN_VALUE

from io import BytesIO
from datetime import datetime
from tempfile import NamedTemporaryFile

from project.settings import env
from rp.utils import cleanup_url


ARTICLE_SCORE_THRESHOLD = 3

STATUS_CHOICES = (
    ("NEW", _("New")),
    ("DRAFT", _("Draft")),
    ("PUBLISHED", _("Published")),
    ("REJECTED", _("Rejected"))
)

LANG_CHOICES = (
    ("FR", _("French")),
    ("EN", _("English")),
    ("NA", _("Other"))
)

URL_HELP_TEXT = """The URL should not contain any marketing tags. We
automatically strip the most known tags."""

TITLE_HELP_TEXT = """Please remove non-necessary parts such as newspapers'
names and leave only the article title."""

EXTRACTS_HELP_TEXT = """Please select short and helpful extracts from the
article content. You should aim at around 500 characters. Use bracket ellipsis
[…] to cut parts not required to understand the context."""


class Article(VoteMixin):
    status = FSMField(default='NEW', choices=STATUS_CHOICES, protected=True)

    url = models.URLField("URL", help_text=URL_HELP_TEXT)
    lang = models.CharField(
        _("Language"), choices=LANG_CHOICES, default="NA", max_length=50)
    metadata = models.TextField(
        _("Opengraph metadata"), blank=True, null=True)
    screenshot = models.ImageField(
        _("Article screenshot"), blank=True, null=True)
    title = models.CharField(
        _("Article title"), max_length=255, default="",
        help_text=TITLE_HELP_TEXT)
    website = models.CharField(_("Website"), max_length=255, default="")
    extracts = models.TextField(
        _("Content extracts"), blank=True, null=True,
        help_text=EXTRACTS_HELP_TEXT)

    created_at = models.DateTimeField(_("Creation date"), auto_now_add=True)
    created_by = models.CharField(max_length=255, null=True)
    updated_at = models.DateTimeField(_("Last update"), auto_now=True)
    published_at = models.DateTimeField(
        _("Publication date"), blank=True, null=True)

    #: priority: True if article have priority
    priority = models.BooleanField(default=False)

    tags = TaggableManager(blank=True)

    class Meta:
        verbose_name = _("Article")
        verbose_name_plural = _("Articles")
        permissions = (
            ("can_change_status", "Can change article status"),
            ("can_change_priority", "Can change article priority"),
            ("can_vote", "Can vote articles"),
            ("can_edit", "Can edit articles")
        )

        ordering = ["-published_at", "-updated_at", "-created_at"]

    def __str__(self):
        return self.title

    # Finite state logic

    @transition(field=status, source='DRAFT', target='PUBLISHED',
                permission="rp.can_change_status")
    def publish(self):
        self.published_at = datetime.now()

    @transition(field=status, source='NEW', target='DRAFT',
                permission="rp.can_change_status")
    def recover(self):
        pass

    @transition(field=status, source=['NEW', 'DRAFT'], target='REJECTED',
                permission="rp.can_change_status")
    def reject(self):
        pass

    @transition(field=status, source='DRAFT', target='DRAFT',
                permission="rp.can_change_priority")
    def set_priority(self):
        self.priority = True

    @transition(field=status, source='DRAFT', target='DRAFT',
                permission="rp.can_change_priority")
    def unset_priority(self):
        self.priority = False

    @transition(field=status, source='DRAFT', target='DRAFT')
    @transition(field=status, source='NEW',
                target=RETURN_VALUE('NEW', 'DRAFT'), permission="rp.can_vote")
    def upvote(self, by=None):
        super(Article, self).upvote(by)
        if self.und_score >= ARTICLE_SCORE_THRESHOLD:
            return 'DRAFT'
        else:
            return self.status

    @transition(field=status, source='NEW', target='NEW', permission="rp.can_vote")
    @transition(field=status, source='DRAFT', target='DRAFT',
                permission="rp.can_vote")
    def downvote(self, by=None):
        super(Article, self).downvote(by)

    def add_new_url(url, by=None):
        url = cleanup_url(url)
        article, _ = Article.objects.get_or_create(url=url)

        if article.created_by is None:
            article.created_by = str(by)

        if by is not None:
            article.upvote(by)

        article.save()
        return article

    # Content extraction

    def fetch_content(self):
        if self.lang != "NA":
            article = ArticleParser(url=self.url, language=self.lang.lower())
        else:
            article = ArticleParser(url=self.url)

        article.download()
        article.parse()
        self.title = article.title
        self.extracts = article.text
        self.save()

    def fetch_image(self):
        import requests
        import imghdr

        if self.lang != "NA":
            article = ArticleParser(url=self.url, language=self.lang.lower())
        else:
            article = ArticleParser(url=self.url)

        article.download()
        article.parse()

        img_path = article.meta_img
        if img_path:
            resp = requests.get(img_path, stream=True)
            if resp.status_code == requests.codes.ok:
                fp = BytesIO()
                fp.write(resp.content)

                file_name_ext = imghdr.what(None, resp.content)
                self.screenshot.save(
                    "screenshot-{0}.{1}".format(self.id, file_name_ext),
                    files.File(fp), save=True)



    def fetch_screenshot(self):
        from selenium import webdriver
        from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
        from PIL import Image
        from pyvirtualdisplay import Display

        with NamedTemporaryFile() as f, Display(visible=False, size=(3200, 1800)):
            binary = None
            if hasattr(env, "FIREFOX_BINARY_PATH"):
                binary = FirefoxBinary(env.FIREFOX_BINARY_PATH)

            profile = None
            if hasattr(env, "FIREFOX_PROFILE_PATH"):
                profile = webdriver.FirefoxProfile(env.FIREFOX_PROFILE_PATH)

            driver = webdriver.Firefox(profile, firefox_binary=binary)
            driver.set_window_size(1200, 1800)

            driver.get(self.url)
            driver.save_screenshot(f.name)
            screen = driver.get_screenshot_as_png()
            driver.quit()

            im = Image.open(BytesIO(screen))
            im.thumbnail((240, 360))
            im_io = BytesIO()
            im.save(im_io, format="PNG")
            self.screenshot.save(
                "screenshot-%i" % self.id, ContentFile(im_io.getvalue()),
                save=True)