~sirn/fanboi2

025ad9d8716d5c1a8786a4f7dd6c0e7bd8907745 — Kridsada Thanabulpong 7 years ago b604707 + 163aff3
Merge branch 'develop' into feature/experiment-view2

Conflicts:
	Vagrantfile
	fanboi2/__init__.py
	fanboi2/resources/app/components/quote_popover.coffee
	fanboi2/templates/boards/error.jinja2
	fanboi2/tests/test_formatters.py
	fanboi2/tests/test_views.py
	fanboi2/utils.py
	fanboi2/views.py
M CHANGES.rst => CHANGES.rst +12 -1
@@ 1,6 1,17 @@
0.8.1
0.8.3
=====

- Changed how post processing failures are handle to no longer rely on `Celery <http://www.celeryproject.org>`_'s exceptions.
- Added cross-board reference syntax with the syntax of ">>>/board/topic/anchor" (e.g. ">>>/demo/123/10-11").

0.8.2
-----

- Fixed Akismet never timed out causing post to hang forever.

0.8.1
-----

- Fixed broken debug toolbar in development mode due to Python 3.2.3 bug.
- Removed pyramid_zcml and webtest from application requirements.
- Changed Celery process runner to no longer load Pyramid environment.

M Vagrantfile => Vagrantfile +1 -0
@@ 2,6 2,7 @@ Vagrant.configure("2") do |config|
  config.vm.box = "hashicorp/precise64"
  config.vm.network :forwarded_port, :guest => 80, :host => 8080
  config.vm.network :private_network, :ip => "192.168.200.100"

  config.vm.provision :ansible do |ansible|
    ansible.limit = "all"
    ansible.playbook = "provisioning/site.yml"

M fanboi2/__init__.py => fanboi2/__init__.py +2 -1
@@ 10,7 10,7 @@ from sqlalchemy.engine import engine_from_config
from .cache import cache_region
from .models import DBSession, Base, redis_conn, identity
from .serializers import add_serializer_adapters
from .utils import akismet, json_renderer
from .utils import akismet, json_renderer, dnsbl


def remote_addr(request):


@@ 102,6 102,7 @@ def configure_components(cfg):  # pragma: no cover
    celery.config_from_object(configure_celery(cfg))
    identity.configure_tz(cfg['app.timezone'])
    akismet.configure_key(cfg['akismet.key'])
    dnsbl.configure_providers(cfg['dnsbl.providers'])
    cache_region.configure_from_config(cfg, 'dogpile.')
    cache_region.invalidate()


M fanboi2/formatters.py => fanboi2/formatters.py +39 -1
@@ 170,6 170,20 @@ def format_markdown(context, request, text):

RE_ANCHOR = re.compile(r'%s(\d+)(\-)?(\d+)?' % html.escape('>>'))
TP_ANCHOR = '<a data-number="%s" href="%s" class="anchor">%s</a>'

RE_ANCHOR_CROSS = re.compile(r"""
  %s\/                       # Syntax start
  (\w+)                      # Board name
  (?:\/(\d+))?               # Topic id
  (?:\/(\d+)(\-)?(\d+)?)     # Post id
  ?(\/?)                     # Trailing slash
""" % html.escape('>>>'), re.VERBOSE)
TP_ANCHOR_CROSS = ''.join("""
<a data-board="%s" data-topic="%s" data-number="%s" href="%s" class="anchor">
%s
</a>
""".splitlines())

TP_SHORTENED = ''.join("""
<p class="shortened">
Post shortened. <a href="%s">See full post</a>.


@@ 208,7 222,31 @@ def format_post(context, request, post, shorten=None):
    except AttributeError:  # pragma: no cover
        pass

    # Convert post anchor (>>123) to link.
    # Convert cross anchor (>>>/demo/123/1-10) into link.
    def _anchor_cross(match):
        board = match.groups()[0]
        topic = match.groups()[1] if match.groups()[1] else ''
        anchor = ''.join([m for m in match.groups()[2:-1] if m is not None])
        trail = match.groups()[-1]

        if board and topic:
            args = {'board': board, 'topic': topic, 'query': anchor}
            args['query'] = anchor if anchor else 'recent'
            path = request.route_path('topic_scoped', **args)
        else:
            path = request.route_path('board', board=board)

        text = []
        for part in (board, topic, anchor):
            if part:
                text.append(part)
        text = html.escape(">>>/%s" % '/'.join(text))
        text += str(trail) if trail else ''
        return Markup(TP_ANCHOR_CROSS % (board, topic, anchor, path, text))

    text = RE_ANCHOR_CROSS.sub(_anchor_cross, text)

    # Convert post anchor (>>123) into link.
    def _anchor(match):
        anchor = ''.join([m for m in match.groups() if m is not None])
        return Markup(TP_ANCHOR % (

M fanboi2/tasks.py => fanboi2/tasks.py +11 -20
@@ 2,7 2,7 @@ import transaction
from celery import Celery
from sqlalchemy.exc import IntegrityError
from .models import DBSession, Post, Topic, Board
from .utils import akismet
from .utils import akismet, dnsbl

celery = Celery()



@@ 26,15 26,7 @@ def configure_celery(settings):  # pragma: no cover
    }


class TaskException(Exception):
    pass


class AddTopicException(TaskException):
    pass


@celery.task(throws=(AddTopicException,))
@celery.task()
def add_topic(request, board_id, title, body):
    """Insert a topic to the database.



@@ 51,7 43,10 @@ def add_topic(request, board_id, title, body):
    :rtype: tuple
    """
    if akismet.spam(request, body):
        raise AddTopicException('spam')
        return 'failure', 'spam'

    if dnsbl.listed(request['remote_addr']):
        return 'failure', 'dnsbl'

    with transaction.manager:
        board = DBSession.query(Board).get(board_id)


@@ 62,11 57,7 @@ def add_topic(request, board_id, title, body):
        return 'topic', post.topic_id


class AddPostException(TaskException):
    pass


@celery.task(bind=True, throws=(AddPostException,), max_retries=4)  # 5 total.
@celery.task(bind=True, max_retries=4)  # 5 total.
def add_post(self, request, topic_id, body, bumped):
    """Insert a post to a topic.



@@ 85,16 76,16 @@ def add_post(self, request, topic_id, body, bumped):
    :rtype: tuple
    """
    if akismet.spam(request, body):
        raise AddPostException('spam')
        return 'failure', 'spam'

    with transaction.manager:
        topic = DBSession.query(Topic).get(topic_id)
        if topic.status != "open":
            return 'failure', topic.status

        post = Post(topic=topic, body=body, bumped=bumped)
        post.ip_address = request['remote_addr']

        if topic.status != "open":
            raise AddPostException(topic.status)

        try:
            DBSession.add(post)
            DBSession.flush()

M fanboi2/tests/test_formatters.py => fanboi2/tests/test_formatters.py +34 -0
@@ 212,6 212,7 @@ class TestFormattersWithModel(ModelMixin, unittest.TestCase):
    def test_format_post(self):
        from fanboi2.formatters import format_post
        from markupsafe import Markup
        self.config.add_route('board', '/{board}')
        self.config.add_route('topic_scoped', '/{board}/{topic}/{query}')
        request = self._makeRequest()
        board = self._makeBoard(title="Foobar", slug="foobar")


@@ 219,6 220,14 @@ class TestFormattersWithModel(ModelMixin, unittest.TestCase):
        post1 = self._makePost(topic=topic, body="Hogehoge\nHogehoge")
        post2 = self._makePost(topic=topic, body=">>1")
        post3 = self._makePost(topic=topic, body=">>1-2\nHoge")
        post4 = self._makePost(topic=topic, body=">>>/demo")
        post5 = self._makePost(topic=topic, body=">>>/demo/123")
        post6 = self._makePost(topic=topic, body=">>>/demo/123/100-")
        post7 = self._makePost(topic=topic, body=">>>/demo/123/100-/")
        post8 = self._makePost(topic=topic, body=">>>/demo/123-/100-/")
        post9 = self._makePost(topic=topic, body=">>>/demo/\n>>>/demo/1/")
        post10 = self._makePost(topic=topic, body=">>>/demo//100-/")
        post11 = self._makePost(topic=topic, body=">>>//123-/100-/")
        tests = [
            (post1, "<p>Hogehoge<br>Hogehoge</p>"),
            (post2, "<p><a data-number=\"1\" " +


@@ 227,6 236,31 @@ class TestFormattersWithModel(ModelMixin, unittest.TestCase):
            (post3, "<p><a data-number=\"1-2\" " +
                    "href=\"/foobar/1/1-2\" class=\"anchor\">" +
                    "&gt;&gt;1-2</a><br>Hoge</p>"),
            (post4, "<p><a data-board=\"demo\" data-topic=\"\" " +
                    "data-number=\"\" href=\"/demo\" class=\"anchor\">" +
                    "&gt;&gt;&gt;/demo</a></p>"),
            (post5, "<p><a data-board=\"demo\" data-topic=\"123\" " +
                    "data-number=\"\" href=\"/demo/123/recent\" " +
                    "class=\"anchor\">&gt;&gt;&gt;/demo/123</a></p>"),
            (post6, "<p><a data-board=\"demo\" data-topic=\"123\" " +
                    "data-number=\"100-\" href=\"/demo/123/100-\" " +
                    "class=\"anchor\">&gt;&gt;&gt;/demo/123/100-</a></p>"),
            (post7, "<p><a data-board=\"demo\" data-topic=\"123\" " +
                    "data-number=\"100-\" href=\"/demo/123/100-\" " +
                    "class=\"anchor\">&gt;&gt;&gt;/demo/123/100-/</a></p>"),
            (post8, "<p><a data-board=\"demo\" data-topic=\"123\" " +
                    "data-number=\"\" href=\"/demo/123/recent\" " +
                    "class=\"anchor\">&gt;&gt;&gt;/demo/123</a>-/100-/</p>"),
            (post9, "<p><a data-board=\"demo\" data-topic=\"\" " +
                    "data-number=\"\" href=\"/demo\" class=\"anchor\">" +
                    "&gt;&gt;&gt;/demo/</a><br><a data-board=\"demo\" " +
                    "data-topic=\"1\" data-number=\"\" " +
                    "href=\"/demo/1/recent\" class=\"anchor\">" +
                    "&gt;&gt;&gt;/demo/1/</a></p>"),
            (post10, "<p><a data-board=\"demo\" data-topic=\"\" " +
                     "data-number=\"\" href=\"/demo\" class=\"anchor\">" +
                     "&gt;&gt;&gt;/demo/</a>/100-/</p>"),
            (post11, "<p>&gt;&gt;&gt;//123-/100-/</p>")
        ]
        for source, target in tests:
            self.assertEqual(format_post(None, request, source), Markup(target))

M fanboi2/tests/test_tasks.py => fanboi2/tests/test_tasks.py +21 -15
@@ 25,10 25,10 @@ class TestAddTopicTask(TaskMixin, ModelMixin, unittest.TestCase):
        self.assertEqual(DBSession.query(Topic).get(result.get()[1]), topic)
        self.assertEqual(topic.title, 'Foobar')
        self.assertEqual(topic.posts[0].body, 'Hello, world!')
        self.assertEqual(result.result, ('topic', topic.id))

    @mock.patch('fanboi2.utils.Akismet.spam')
    def test_add_topic_spam(self, akismet):
        from fanboi2.tasks import AddTopicException
        from fanboi2.models import Topic
        akismet.return_value = True
        request = {'remote_addr': '127.0.0.1'}


@@ 36,11 36,22 @@ class TestAddTopicTask(TaskMixin, ModelMixin, unittest.TestCase):
            board = self._makeBoard(title='Foobar', slug='foobar')
            board_id = board.id  # board is not bound outside transaction!
        result = self._makeOne(request, board_id, 'Foobar', 'Hello, world!')
        self.assertFalse(result.successful())
        self.assertTrue(result.successful())
        self.assertEqual(DBSession.query(Topic).count(), 0)
        with self.assertRaises(AddTopicException) as e:
            assert not result.get()
        self.assertEqual(e.exception.args, ('spam',))
        self.assertEqual(result.result, ('failure', 'spam'))

    @mock.patch('fanboi2.utils.Dnsbl.listed')
    def test_add_topic_dnsbl(self, dnsbl):
        from fanboi2.models import Topic
        dnsbl.return_value = True
        request = {'remote_addr': '127.0.0.1'}
        with transaction.manager:
            board = self._makeBoard(title='Foobar', slug='foobar')
            board_id = board.id  # board is not bound outside transaction!
        result = self._makeOne(request, board_id, 'Foobar', 'Hello, world!')
        self.assertTrue(result.successful())
        self.assertEqual(DBSession.query(Topic).count(), 0)
        self.assertEqual(result.result, ('failure', 'dnsbl'))


class TestAddPostTask(TaskMixin, ModelMixin, unittest.TestCase):


@@ 64,11 75,11 @@ class TestAddPostTask(TaskMixin, ModelMixin, unittest.TestCase):
        self.assertEqual(DBSession.query(Post).get(result.get()[1]), post)
        self.assertEqual(post.body, 'Hi!')
        self.assertEqual(post.bumped, True)
        self.assertEqual(result.result, ('post', post.id))

    @mock.patch('fanboi2.utils.Akismet.spam')
    def test_add_post_spam(self, akismet):
        import transaction
        from fanboi2.tasks import AddPostException
        from fanboi2.models import Post
        akismet.return_value = True
        request = {'remote_addr': '127.0.0.1'}


@@ 77,15 88,12 @@ class TestAddPostTask(TaskMixin, ModelMixin, unittest.TestCase):
            topic = self._makeTopic(board=board, title='Hello, world!')
            topic_id = topic.id  # topic is not bound outside transaction!
        result = self._makeOne(request, topic_id, 'Hi!', True)
        self.assertFalse(result.successful())
        self.assertTrue(result.successful())
        self.assertEqual(DBSession.query(Post).count(), 0)
        with self.assertRaises(AddPostException) as e:
            assert not result.get()
        self.assertEqual(e.exception.args, ('spam',))
        self.assertEqual(result.result, ('failure', 'spam'))

    def test_add_post_locked(self):
        import transaction
        from fanboi2.tasks import AddPostException
        from fanboi2.models import Post
        request = {'remote_addr': '127.0.0.1'}
        with transaction.manager:


@@ 96,11 104,9 @@ class TestAddPostTask(TaskMixin, ModelMixin, unittest.TestCase):
                status='locked')
            topic_id = topic.id  # topic is not bound outside transaction!
        result = self._makeOne(request, topic_id, 'Hi!', True)
        self.assertFalse(result.successful())
        self.assertTrue(result.successful())
        self.assertEqual(DBSession.query(Post).count(), 0)
        with self.assertRaises(AddPostException) as e:
            assert not result.get()
        self.assertEqual(e.exception.args, ('locked',))
        self.assertEqual(result.result, ('failure', 'locked'))

    def test_add_post_retry(self):
        import transaction

M fanboi2/tests/test_utils.py => fanboi2/tests/test_utils.py +59 -0
@@ 34,6 34,55 @@ class TestRequestSerializer(unittest.TestCase):
        self.assertEqual(self._getTargetFunction()(request), request)


class TestDnsBl(unittest.TestCase):

    def _makeOne(self, providers=None):
        from fanboi2.utils import dnsbl
        if providers is None:
            providers = ['xbl.spamhaus.org']
        dnsbl.configure_providers(providers)
        return dnsbl

    def test_init(self):
        dnsbl = self._makeOne()
        self.assertEqual(dnsbl.providers, ['xbl.spamhaus.org'])

    def test_init_no_providers(self):
        dnsbl = self._makeOne(providers=[])
        self.assertEqual(dnsbl.providers, [])

    def test_init_string_providers(self):
        dnsbl = self._makeOne(providers='xbl.spamhaus.org tor.ahbl.org')
        self.assertEqual(dnsbl.providers, ['xbl.spamhaus.org', 'tor.ahbl.org'])

    @mock.patch('socket.gethostbyname')
    def test_listed(self, lookup_call):
        lookup_call.return_value = '127.0.0.2'
        dnsbl = self._makeOne()
        self.assertEqual(dnsbl.listed('10.0.100.254'), True)
        lookup_call.assert_called_with('254.100.0.10.xbl.spamhaus.org.')

    @mock.patch('socket.gethostbyname')
    def test_listed_unlisted(self, lookup_call):
        import socket
        lookup_call.side_effect = socket.gaierror('foobar')
        dnsbl = self._makeOne()
        self.assertEqual(dnsbl.listed('10.0.100.1'), False)
        lookup_call.assert_called_with('1.100.0.10.xbl.spamhaus.org.')

    @mock.patch('socket.gethostbyname')
    def test_listed_invalid(self, lookup_call):
        lookup_call.return_value = '192.168.1.1'
        dnsbl = self._makeOne()
        self.assertEqual(dnsbl.listed('10.0.100.2'), False)

    @mock.patch('socket.gethostbyname')
    def test_listed_malformed(self, lookup_call):
        lookup_call.return_value = 'foobarbaz'
        dnsbl = self._makeOne()
        self.assertEqual(dnsbl.listed('10.0.100.2'), False)


class TestAkismet(unittest.TestCase):

    def _makeOne(self, key='hogehoge'):


@@ 76,6 125,7 @@ class TestAkismet(unittest.TestCase):
            'https://hogehoge.rest.akismet.com/1.1/comment-check',
            headers=mock.ANY,
            data=mock.ANY,
            timeout=mock.ANY,
        )

    @mock.patch('requests.post')


@@ 88,8 138,17 @@ class TestAkismet(unittest.TestCase):
            'https://hogehoge.rest.akismet.com/1.1/comment-check',
            headers=mock.ANY,
            data=mock.ANY,
            timeout=mock.ANY,
        )

    @mock.patch('requests.post')
    def test_spam_timeout(self, api_call):
        import requests
        request = self._makeRequest()
        akismet = self._makeOne()
        api_call.side_effect = requests.Timeout('connection timed out')
        self.assertEqual(akismet.spam(request, 'buy viagra'), False)

    # noinspection PyTypeChecker
    @mock.patch('requests.post')
    def test_spam_no_key(self, api_call):

M fanboi2/utils.py => fanboi2/utils.py +49 -11
@@ 1,6 1,8 @@
import datetime
import hashlib
import requests
import socket
from IPy import IP
from pyramid.renderers import JSON
from sqlalchemy.orm import Query
from .models import redis_conn


@@ 28,6 30,37 @@ def serialize_request(request):
    }


class Dnsbl(object) :
    """Utility class for checking IP address against DNSBL providers."""

    def __init__(self):
        self.providers = []

    def configure_providers(self, providers):
        if isinstance(providers, str):
            providers = providers.split()
        self.providers = providers

    def listed(self, ip_address):
        """Returns :type:`True` if the given IP address is listed in the
        DNSBL providers. Returns :type:`False` if not listed or no DNSBL
        providers present.
        """
        if self.providers:
            for provider in self.providers:
                try:
                    check = '.'.join(reversed(ip_address.split('.')))
                    res = socket.gethostbyname("%s.%s." % (check, provider))
                    if IP(res).make_net('255.0.0.0') == IP('127.0.0.0/8'):
                        return True
                except (socket.gaierror, ValueError):
                    continue
        return False


dnsbl = Dnsbl()


class Akismet(object):
    """Basic integration between Pyramid and Akismet."""



@@ 57,11 90,13 @@ class Akismet(object):
        return requests.post(
            'https://%s.rest.akismet.com/1.1/%s' % (self.key, name),
            headers={'User-Agent': "Fanboi2/%s | Akismet/0.1" % __VERSION__},
            data=data)
            data=data,
            timeout=2)

    def spam(self, request, message):
        """Returns :type:`True` if `message` is spam. Always returns
        :type:`False` if Akismet key is not set.
        :type:`False` if Akismet key is not set or the request to Akismet
        was timed out.

        :param request: A :class:`pyramid.request.Request` object.
        :param message: A :type:`str` to identify.


@@ 72,15 107,18 @@ class Akismet(object):
        """
        if self.key:
            request = serialize_request(request)
            return self._api_post('comment-check', data={
                'blog': request['application_url'],
                'user_ip': request['remote_addr'],
                'user_agent': request['user_agent'],
                'referrer': request['referrer'],
                'permalink': request['url'],
                'comment_type': 'comment',
                'comment_content': message,
            }).content == b'true'
            try:
                return self._api_post('comment-check', data={
                    'blog': request['application_url'],
                    'user_ip': request['remote_addr'],
                    'user_agent': request['user_agent'],
                    'referrer': request['referrer'],
                    'permalink': request['url'],
                    'comment_type': 'comment',
                    'comment_content': message,
                }).content == b'true'
            except requests.Timeout:
                return False
        return False



M provisioning/files/srv/settings.ini.j2 => provisioning/files/srv/settings.ini.j2 +1 -0
@@ 27,6 27,7 @@ sqlalchemy.url = postgresql://{{ db_user }}:{{ db_pass }}@{{ ansible_default_ipv
redis.url = redis://{{ ansible_default_ipv4.address }}:6379/0
celery.broker = redis://{{ ansible_default_ipv4.address }}:6379/1
akismet.key = {{ akismet_key }}
dnsbl.providers = {{ dnsbl_providers }}

dogpile.backend = dogpile.cache.memcached
dogpile.arguments.url = {{ ansible_default_ipv4.address }}:11211

M provisioning/group_vars/all.yml => provisioning/group_vars/all.yml +1 -0
@@ 6,6 6,7 @@ ssl: 0
timezone: "Asia/Bangkok"
csrf_secret: "changeme"
session_secret: "changeme"
dnsbl_providers: "zen.spamhaus.org"
akismet_key: ""

# Application paths

M provisioning/site.yml => provisioning/site.yml +7 -7
@@ 138,9 138,9 @@

    # pip 1.5 and virtualenv 1.11 is incompatible with system setuptools.
    - name: install python3 requisites
      command: |
        easy_install3 {{item}}
        creates=/usr/local/bin/{{item}}-3.2
      command: "easy_install3 {{item}}"
      args:
        creates: "/usr/local/bin/{{item}}-3.2"
      with_items:
        - pip==1.4.1
        - virtualenv==1.10.1


@@ 190,14 190,14 @@

    - name: setup virtualenv for application directory
      sudo_user: "{{user}}"
      command: |
        virtualenv -p python3 {{virtualenv}}
        creates={{virtualenv}}
      command: "virtualenv -p python3 {{virtualenv}}"
      args:
        creates: "{{virtualenv}}"

    - name: install application in development mode
      sudo_user: "{{user}}"
      command: |
        env LANG=en_US.UTF-8 {{pip32}} install -e . --use-mirrors
        env LANG=en_US.UTF-8 {{pip32}} install -e {{root}} --use-mirrors
        chdir={{root}}
      when: development|int == 1 or fanboi2_cloned.changed


M setup.py => setup.py +1 -1
@@ 46,7 46,7 @@ requires = [
    ]

setup(name='fanboi2',
      version='0.8.1',
      version='0.8.3',
      description='fanboi2',
      long_description=readme + '\n\n' + changes,
      classifiers=[