~sirn/fanboi2

43c2028271989d2b6338fe7ccd62713297145b93 — Kridsada Thanabulpong 7 years ago 9a0e916 + 8b56ac6
Merge branch 'feature/proxy-autoban' into develop
M CHANGES.rst => CHANGES.rst +11 -1
@@ 1,6 1,16 @@
0.8.1
0.8.3
=====

- Changed how post processing failures are handle to no longer rely on `Celery <http://www.celeryproject.org>`_'s exceptions.

0.8.2
-----

- Fixed Akismet never timed out causing post to hang forever.

0.8.1
-----

- Fixed broken debug toolbar in development mode due to Python 3.2.3 bug.
- Removed pyramid_zcml and webtest from application requirements.
- Changed Celery process runner to no longer load Pyramid environment.

M fanboi2/__init__.py => fanboi2/__init__.py +2 -1
@@ 11,7 11,7 @@ from sqlalchemy import engine_from_config
from .cache import cache_region, Jinja2CacheExtension
from .formatters import *
from .models import DBSession, Base, redis_conn, identity
from .utils import akismet
from .utils import akismet, dnsbl


def remote_addr(request):


@@ 70,6 70,7 @@ def configure_components(cfg):  # pragma: no cover
    celery.config_from_object(configure_celery(cfg))
    identity.configure_tz(cfg['app.timezone'])
    akismet.configure_key(cfg['akismet.key'])
    dnsbl.configure_providers(cfg['dnsbl.providers'])
    cache_region.configure_from_config(cfg, 'dogpile.')
    cache_region.invalidate()


M fanboi2/tasks.py => fanboi2/tasks.py +11 -20
@@ 2,7 2,7 @@ import transaction
from celery import Celery
from sqlalchemy.exc import IntegrityError
from .models import DBSession, Post, Topic, Board
from .utils import akismet
from .utils import akismet, dnsbl

celery = Celery()



@@ 20,19 20,14 @@ def configure_celery(settings):  # pragma: no cover
    }


class TaskException(Exception):
    pass


class AddTopicException(TaskException):
    pass


@celery.task(throws=(AddTopicException,))
@celery.task()
def add_topic(request, board_id, title, body):
    """Insert a topic to the database."""
    if akismet.spam(request, body):
        raise AddTopicException('spam')
        return 'failure', 'spam'

    if dnsbl.listed(request['remote_addr']):
        return 'failure', 'dnsbl'

    with transaction.manager:
        board = DBSession.query(Board).get(board_id)


@@ 43,24 38,20 @@ def add_topic(request, board_id, title, body):
        return 'topic', post.topic_id


class AddPostException(TaskException):
    pass


@celery.task(bind=True, throws=(AddPostException,), max_retries=4)  # 5 total.
@celery.task(bind=True, max_retries=4)  # 5 total.
def add_post(self, request, topic_id, body, bumped):
    """Insert a post to a topic."""
    if akismet.spam(request, body):
        raise AddPostException('spam')
        return 'failure', 'spam'

    with transaction.manager:
        topic = DBSession.query(Topic).get(topic_id)
        if topic.status != "open":
            return 'failure', topic.status

        post = Post(topic=topic, body=body, bumped=bumped)
        post.ip_address = request['remote_addr']

        if topic.status != "open":
            raise AddPostException(topic.status)

        try:
            DBSession.add(post)
            DBSession.flush()

M fanboi2/templates/boards/error.jinja2 => fanboi2/templates/boards/error.jinja2 +4 -0
@@ 19,6 19,10 @@
                        <h3>I'm sorry, Dave. I'm afraid I can't do that.</h3>
                        <p>This mission is too important for me to allow you to jeopardize it.</p>
                        <p class="fineprint">Your message has been identified as spam and therefore rejected.</p>
                    {% elif error == "dnsbl" %}
                        <h3>Rainbows! Rainbows everywhere!</h3>
                        <p>What about us finding the beginning of the rainbow together.</p>
                        <p class="fineprint">Your IP address is listed in DNSBL and therefore rejected.</p>
                    {% endif %}
                {% endif %}
            </div>

M fanboi2/tests/test_tasks.py => fanboi2/tests/test_tasks.py +21 -15
@@ 25,10 25,10 @@ class TestAddTopicTask(TaskMixin, ModelMixin, unittest.TestCase):
        self.assertEqual(DBSession.query(Topic).get(result.get()[1]), topic)
        self.assertEqual(topic.title, 'Foobar')
        self.assertEqual(topic.posts[0].body, 'Hello, world!')
        self.assertEqual(result.result, ('topic', topic.id))

    @mock.patch('fanboi2.utils.Akismet.spam')
    def test_add_topic_spam(self, akismet):
        from fanboi2.tasks import AddTopicException
        from fanboi2.models import Topic
        akismet.return_value = True
        request = {'remote_addr': '127.0.0.1'}


@@ 36,11 36,22 @@ class TestAddTopicTask(TaskMixin, ModelMixin, unittest.TestCase):
            board = self._makeBoard(title='Foobar', slug='foobar')
            board_id = board.id  # board is not bound outside transaction!
        result = self._makeOne(request, board_id, 'Foobar', 'Hello, world!')
        self.assertFalse(result.successful())
        self.assertTrue(result.successful())
        self.assertEqual(DBSession.query(Topic).count(), 0)
        with self.assertRaises(AddTopicException) as e:
            assert not result.get()
        self.assertEqual(e.exception.args, ('spam',))
        self.assertEqual(result.result, ('failure', 'spam'))

    @mock.patch('fanboi2.utils.Dnsbl.listed')
    def test_add_topic_dnsbl(self, dnsbl):
        from fanboi2.models import Topic
        dnsbl.return_value = True
        request = {'remote_addr': '127.0.0.1'}
        with transaction.manager:
            board = self._makeBoard(title='Foobar', slug='foobar')
            board_id = board.id  # board is not bound outside transaction!
        result = self._makeOne(request, board_id, 'Foobar', 'Hello, world!')
        self.assertTrue(result.successful())
        self.assertEqual(DBSession.query(Topic).count(), 0)
        self.assertEqual(result.result, ('failure', 'dnsbl'))


class TestAddPostTask(TaskMixin, ModelMixin, unittest.TestCase):


@@ 64,11 75,11 @@ class TestAddPostTask(TaskMixin, ModelMixin, unittest.TestCase):
        self.assertEqual(DBSession.query(Post).get(result.get()[1]), post)
        self.assertEqual(post.body, 'Hi!')
        self.assertEqual(post.bumped, True)
        self.assertEqual(result.result, ('post', post.id))

    @mock.patch('fanboi2.utils.Akismet.spam')
    def test_add_post_spam(self, akismet):
        import transaction
        from fanboi2.tasks import AddPostException
        from fanboi2.models import Post
        akismet.return_value = True
        request = {'remote_addr': '127.0.0.1'}


@@ 77,15 88,12 @@ class TestAddPostTask(TaskMixin, ModelMixin, unittest.TestCase):
            topic = self._makeTopic(board=board, title='Hello, world!')
            topic_id = topic.id  # topic is not bound outside transaction!
        result = self._makeOne(request, topic_id, 'Hi!', True)
        self.assertFalse(result.successful())
        self.assertTrue(result.successful())
        self.assertEqual(DBSession.query(Post).count(), 0)
        with self.assertRaises(AddPostException) as e:
            assert not result.get()
        self.assertEqual(e.exception.args, ('spam',))
        self.assertEqual(result.result, ('failure', 'spam'))

    def test_add_post_locked(self):
        import transaction
        from fanboi2.tasks import AddPostException
        from fanboi2.models import Post
        request = {'remote_addr': '127.0.0.1'}
        with transaction.manager:


@@ 96,11 104,9 @@ class TestAddPostTask(TaskMixin, ModelMixin, unittest.TestCase):
                status='locked')
            topic_id = topic.id  # topic is not bound outside transaction!
        result = self._makeOne(request, topic_id, 'Hi!', True)
        self.assertFalse(result.successful())
        self.assertTrue(result.successful())
        self.assertEqual(DBSession.query(Post).count(), 0)
        with self.assertRaises(AddPostException) as e:
            assert not result.get()
        self.assertEqual(e.exception.args, ('locked',))
        self.assertEqual(result.result, ('failure', 'locked'))

    def test_add_post_retry(self):
        import transaction

M fanboi2/tests/test_utils.py => fanboi2/tests/test_utils.py +49 -0
@@ 34,6 34,55 @@ class TestRequestSerializer(unittest.TestCase):
        self.assertEqual(self._getTargetFunction()(request), request)


class TestDnsBl(unittest.TestCase):

    def _makeOne(self, providers=None):
        from fanboi2.utils import dnsbl
        if providers is None:
            providers = ['xbl.spamhaus.org']
        dnsbl.configure_providers(providers)
        return dnsbl

    def test_init(self):
        dnsbl = self._makeOne()
        self.assertEqual(dnsbl.providers, ['xbl.spamhaus.org'])

    def test_init_no_providers(self):
        dnsbl = self._makeOne(providers=[])
        self.assertEqual(dnsbl.providers, [])

    def test_init_string_providers(self):
        dnsbl = self._makeOne(providers='xbl.spamhaus.org tor.ahbl.org')
        self.assertEqual(dnsbl.providers, ['xbl.spamhaus.org', 'tor.ahbl.org'])

    @mock.patch('socket.gethostbyname')
    def test_listed(self, lookup_call):
        lookup_call.return_value = '127.0.0.2'
        dnsbl = self._makeOne()
        self.assertEqual(dnsbl.listed('10.0.100.254'), True)
        lookup_call.assert_called_with('254.100.0.10.xbl.spamhaus.org.')

    @mock.patch('socket.gethostbyname')
    def test_listed_unlisted(self, lookup_call):
        import socket
        lookup_call.side_effect = socket.gaierror('foobar')
        dnsbl = self._makeOne()
        self.assertEqual(dnsbl.listed('10.0.100.1'), False)
        lookup_call.assert_called_with('1.100.0.10.xbl.spamhaus.org.')

    @mock.patch('socket.gethostbyname')
    def test_listed_invalid(self, lookup_call):
        lookup_call.return_value = '192.168.1.1'
        dnsbl = self._makeOne()
        self.assertEqual(dnsbl.listed('10.0.100.2'), False)

    @mock.patch('socket.gethostbyname')
    def test_listed_malformed(self, lookup_call):
        lookup_call.return_value = 'foobarbaz'
        dnsbl = self._makeOne()
        self.assertEqual(dnsbl.listed('10.0.100.2'), False)


class TestAkismet(unittest.TestCase):

    def _makeOne(self, key='hogehoge'):

M fanboi2/tests/test_views.py => fanboi2/tests/test_views.py +6 -12
@@ 138,10 138,8 @@ class TestBaseTaskView(ViewMixin, ModelMixin, unittest.TestCase):

    @mock.patch('fanboi2.tasks.celery.AsyncResult')
    def test_get_dispatch_failure(self, async):
        from celery.states import FAILURE
        from fanboi2.tasks import TaskException
        exception = TaskException('foobar')
        async.return_value = DummyAsyncResult(FAILURE, None, exception)
        from celery.states import SUCCESS
        async.return_value = DummyAsyncResult(SUCCESS, ('failure', 'foobar'))
        request = self._GET({'task': '1234'})
        view = self._makeOne()(request)
        with mock.patch.object(view, 'GET_failure') as get_call:


@@ 291,10 289,8 @@ class TestBoardNewView(ViewMixin, ModelMixin, unittest.TestCase):

    @mock.patch('fanboi2.tasks.celery.AsyncResult')
    def test_get_failure(self, async):
        from celery.states import FAILURE
        from fanboi2.tasks import TaskException
        exception = TaskException('foobar')
        async.return_value = DummyAsyncResult(FAILURE, None, exception)
        from celery.states import SUCCESS
        async.return_value = DummyAsyncResult(SUCCESS, ('failure', 'foobar'))
        self.config.testing_add_renderer('boards/error.jinja2')
        request = self._GET({'task': '1234'})
        response = self._getTargetClass()(request)()


@@ 436,10 432,8 @@ class TestTopicView(ViewMixin, ModelMixin, unittest.TestCase):

    @mock.patch('fanboi2.tasks.celery.AsyncResult')
    def test_get_failure(self, async):
        from celery.states import FAILURE
        from fanboi2.tasks import TaskException
        exception = TaskException('foobar')
        async.return_value = DummyAsyncResult(FAILURE, None, exception)
        from celery.states import SUCCESS
        async.return_value = DummyAsyncResult(SUCCESS, ('failure', 'foobar'))
        self.config.testing_add_renderer('topics/error.jinja2')
        request = self._GET({'task': '1234'})
        response = self._getTargetClass()(request)()

M fanboi2/utils.py => fanboi2/utils.py +33 -0
@@ 1,5 1,7 @@
import hashlib
import requests
import socket
from IPy import IP
from .models import redis_conn
from .version import __VERSION__



@@ 19,6 21,37 @@ def serialize_request(request):
    }


class Dnsbl(object) :
    """Utility class for checking IP address against DNSBL providers."""

    def __init__(self):
        self.providers = []

    def configure_providers(self, providers):
        if isinstance(providers, str):
            providers = providers.split()
        self.providers = providers

    def listed(self, ip_address):
        """Returns :type:`True` if the given IP address is listed in the
        DNSBL providers. Returns :type:`False` if not listed or no DNSBL
        providers present.
        """
        if self.providers:
            for provider in self.providers:
                try:
                    check = '.'.join(reversed(ip_address.split('.')))
                    res = socket.gethostbyname("%s.%s." % (check, provider))
                    if IP(res).make_net('255.0.0.0') == IP('127.0.0.0/8'):
                        return True
                except (socket.gaierror, ValueError):
                    continue
        return False


dnsbl = Dnsbl()


class Akismet(object):
    """Basic integration between Pyramid and Akismet."""


M fanboi2/views.py => fanboi2/views.py +7 -9
@@ 8,7 8,7 @@ from sqlalchemy.orm import undefer
from sqlalchemy.orm.exc import NoResultFound
from .forms import TopicForm, PostForm
from .models import Topic, Post, Board, DBSession
from .tasks import celery, add_topic, add_post, TaskException
from .tasks import celery, add_topic, add_post
from .utils import RateLimiter, serialize_request




@@ 74,14 74,12 @@ class BaseTaskView(object):
            task = celery.AsyncResult(self.request.params['task'])

            if task.state == states.SUCCESS:
                obj = self._serialize(task.get())
                return self.GET_success(obj)

            elif task.state == states.FAILURE:
                try:
                    task.get()
                except TaskException as exc:
                    return self.GET_failure(exc.args[0])
                result = task.get()
                if result[0] == 'failure':
                    return self.GET_failure(result[1])
                else:
                    obj = self._serialize(result)
                    return self.GET_success(obj)

            else:
                return self.GET_task()

M provisioning/files/srv/settings.ini.j2 => provisioning/files/srv/settings.ini.j2 +1 -0
@@ 27,6 27,7 @@ sqlalchemy.url = postgresql://{{ db_user }}:{{ db_pass }}@{{ ansible_default_ipv
redis.url = redis://{{ ansible_default_ipv4.address }}:6379/0
celery.broker = redis://{{ ansible_default_ipv4.address }}:6379/1
akismet.key = {{ akismet_key }}
dnsbl.providers = {{ dnsbl_providers }}

dogpile.backend = dogpile.cache.memcached
dogpile.arguments.url = {{ ansible_default_ipv4.address }}:11211

M provisioning/group_vars/all.yml => provisioning/group_vars/all.yml +1 -0
@@ 6,6 6,7 @@ ssl: 0
timezone: "Asia/Bangkok"
csrf_secret: "changeme"
session_secret: "changeme"
dnsbl_providers: "xbl.spamhaus.org"
akismet_key: ""

# Application paths

M provisioning/site.yml => provisioning/site.yml +6 -6
@@ 138,9 138,9 @@

    # pip 1.5 and virtualenv 1.11 is incompatible with system setuptools.
    - name: install python3 requisites
      command: |
        easy_install3 {{item}}
        creates=/usr/local/bin/{{item}}-3.2
      command: "easy_install3 {{item}}"
      args:
        creates: "/usr/local/bin/{{item}}-3.2"
      with_items:
        - pip==1.4.1
        - virtualenv==1.10.1


@@ 189,9 189,9 @@

    - name: setup virtualenv for application directory
      sudo_user: "{{user}}"
      command: |
        virtualenv -p python3 {{virtualenv}}
        creates={{virtualenv}}
      command: "virtualenv -p python3 {{virtualenv}}"
      args:
        creates: "{{virtualenv}}"

    - name: install application in development mode
      sudo_user: "{{user}}"

M setup.py => setup.py +1 -1
@@ 47,7 47,7 @@ requires = [
    ]

setup(name='fanboi2',
      version='0.8.2',
      version='0.8.3',
      description='fanboi2',
      long_description=readme + '\n\n' + changes,
      classifiers=[