~cedric/newspipe

c0d48f8a060fa30107183ad024e8c03cfee0eb26 — Cédric Bonhomme 1 year, 2 months ago 7866813
added a little black magic
54 files changed, 1947 insertions(+), 1408 deletions(-)

M newspipe/bootstrap.py
M newspipe/conf.py
M newspipe/crawler/default_crawler.py
M newspipe/lib/article_utils.py
M newspipe/lib/data.py
M newspipe/lib/feed_utils.py
M newspipe/lib/misc_utils.py
M newspipe/lib/utils.py
M newspipe/manager.py
M newspipe/notifications/emails.py
M newspipe/notifications/notifications.py
M newspipe/runserver.py
M newspipe/web/controllers/__init__.py
M newspipe/web/controllers/abstract.py
M newspipe/web/controllers/article.py
M newspipe/web/controllers/bookmark.py
M newspipe/web/controllers/category.py
M newspipe/web/controllers/feed.py
M newspipe/web/controllers/icon.py
M newspipe/web/controllers/user.py
M newspipe/web/decorators.py
M newspipe/web/forms.py
M newspipe/web/lib/user_utils.py
M newspipe/web/lib/view_utils.py
M newspipe/web/models/__init__.py
M newspipe/web/models/article.py
M newspipe/web/models/bookmark.py
M newspipe/web/models/category.py
M newspipe/web/models/feed.py
M newspipe/web/models/right_mixin.py
M newspipe/web/models/role.py
M newspipe/web/models/tag.py
M newspipe/web/models/user.py
M newspipe/web/views/__init__.py
M newspipe/web/views/admin.py
M newspipe/web/views/api/v2/__init__.py
M newspipe/web/views/api/v2/article.py
M newspipe/web/views/api/v2/category.py
M newspipe/web/views/api/v2/common.py
M newspipe/web/views/api/v2/feed.py
M newspipe/web/views/api/v3/__init__.py
M newspipe/web/views/api/v3/article.py
M newspipe/web/views/api/v3/common.py
M newspipe/web/views/api/v3/feed.py
M newspipe/web/views/article.py
M newspipe/web/views/bookmark.py
M newspipe/web/views/category.py
M newspipe/web/views/common.py
M newspipe/web/views/feed.py
M newspipe/web/views/home.py
M newspipe/web/views/icon.py
M newspipe/web/views/session_mgmt.py
M newspipe/web/views/user.py
M newspipe/web/views/views.py
M newspipe/bootstrap.py => newspipe/bootstrap.py +37 -24
@@ 10,16 10,27 @@ import flask_restless
from urllib.parse import urlsplit


def set_logging(log_path=None, log_level=logging.INFO, modules=(),
                log_format='%(asctime)s %(levelname)s %(message)s'):
def set_logging(
    log_path=None,
    log_level=logging.INFO,
    modules=(),
    log_format="%(asctime)s %(levelname)s %(message)s",
):
    if not modules:
        modules = ('root', 'bootstrap', 'runserver',
                   'web', 'crawler.default_crawler', 'manager', 'plugins')
        modules = (
            "root",
            "bootstrap",
            "runserver",
            "web",
            "crawler.default_crawler",
            "manager",
            "plugins",
        )
    if log_path:
        if not os.path.exists(os.path.dirname(log_path)):
            os.makedirs(os.path.dirname(log_path))
        if not os.path.exists(log_path):
            open(log_path, 'w').close()
            open(log_path, "w").close()
        handler = logging.FileHandler(log_path)
    else:
        handler = logging.StreamHandler()


@@ 32,39 43,40 @@ def set_logging(log_path=None, log_level=logging.INFO, modules=(),
            handler.setLevel(log_level)
        logger.setLevel(log_level)


from flask import Flask
from flask_sqlalchemy import SQLAlchemy

# Create Flask application
application = Flask('web')
if os.environ.get('Newspipe_TESTING', False) == 'true':
application = Flask("web")
if os.environ.get("Newspipe_TESTING", False) == "true":
    application.debug = logging.DEBUG
    application.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
    application.config['TESTING'] = True
    application.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
    application.config["TESTING"] = True
else:
    application.debug = conf.LOG_LEVEL <= logging.DEBUG
    application.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
    application.config['SQLALCHEMY_DATABASE_URI'] \
            = conf.SQLALCHEMY_DATABASE_URI
    if 'postgres' in conf.SQLALCHEMY_DATABASE_URI:
        application.config['SQLALCHEMY_POOL_SIZE'] = 15
        application.config['SQLALCHEMY_MAX_OVERFLOW'] = 0
    application.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
    application.config["SQLALCHEMY_DATABASE_URI"] = conf.SQLALCHEMY_DATABASE_URI
    if "postgres" in conf.SQLALCHEMY_DATABASE_URI:
        application.config["SQLALCHEMY_POOL_SIZE"] = 15
        application.config["SQLALCHEMY_MAX_OVERFLOW"] = 0

scheme, domain, _, _, _ = urlsplit(conf.PLATFORM_URL)
application.config['SERVER_NAME'] = domain
application.config['PREFERRED_URL_SCHEME'] = scheme
application.config["SERVER_NAME"] = domain
application.config["PREFERRED_URL_SCHEME"] = scheme

set_logging(conf.LOG_PATH, log_level=conf.LOG_LEVEL)

# Create secrey key so we can use sessions
application.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None)
if not application.config['SECRET_KEY']:
    application.config['SECRET_KEY'] = os.urandom(12)
application.config["SECRET_KEY"] = getattr(conf, "WEBSERVER_SECRET", None)
if not application.config["SECRET_KEY"]:
    application.config["SECRET_KEY"] = os.urandom(12)

application.config['SECURITY_PASSWORD_SALT'] = getattr(conf,
                                                'SECURITY_PASSWORD_SALT', None)
if not application.config['SECURITY_PASSWORD_SALT']:
    application.config['SECURITY_PASSWORD_SALT'] = os.urandom(12)
application.config["SECURITY_PASSWORD_SALT"] = getattr(
    conf, "SECURITY_PASSWORD_SALT", None
)
if not application.config["SECURITY_PASSWORD_SALT"]:
    application.config["SECURITY_PASSWORD_SALT"] = os.urandom(12)

db = SQLAlchemy(application)



@@ 74,5 86,6 @@ manager = flask_restless.APIManager(application, flask_sqlalchemy_db=db)

def populate_g():
    from flask import g

    g.db = db
    g.app = application

M newspipe/conf.py => newspipe/conf.py +60 -63
@@ 10,41 10,36 @@ import logging

BASE_DIR = os.path.abspath(os.path.dirname(__file__))
PATH = os.path.abspath(".")
API_ROOT = '/api/v2.0'
API_ROOT = "/api/v2.0"

# available languages
LANGUAGES = {
    'en': 'English',
    'fr': 'French'
}
LANGUAGES = {"en": "English", "fr": "French"}

TIME_ZONE = {
    "en": "US/Eastern",
    "fr": "Europe/Paris"
}
TIME_ZONE = {"en": "US/Eastern", "fr": "Europe/Paris"}

DEFAULTS = {"platform_url": "https://www.newspipe.org/",
            "self_registration": "false",
            "cdn_address": "",
            "admin_email": "info@newspipe.org",
            "token_validity_period": "3600",
            "default_max_error": "3",
            "log_path": "newspipe.log",
            "log_level": "info",
            "secret_key": "",
            "security_password_salt": "",
            "enabled": "false",
            "notification_email": "info@newspipe.org",
            "tls": "false",
            "ssl": "true",
            "host": "0.0.0.0",
            "port": "5000",
            "crawling_method": "default",
            "crawler_user_agent": "Newspipe (https://github.com/newspipe)",
            "crawler_timeout": "30",
            "crawler_resolv": "false",
            "feed_refresh_interval": "120"
            }
DEFAULTS = {
    "platform_url": "https://www.newspipe.org/",
    "self_registration": "false",
    "cdn_address": "",
    "admin_email": "info@newspipe.org",
    "token_validity_period": "3600",
    "default_max_error": "3",
    "log_path": "newspipe.log",
    "log_level": "info",
    "secret_key": "",
    "security_password_salt": "",
    "enabled": "false",
    "notification_email": "info@newspipe.org",
    "tls": "false",
    "ssl": "true",
    "host": "0.0.0.0",
    "port": "5000",
    "crawling_method": "default",
    "crawler_user_agent": "Newspipe (https://github.com/newspipe)",
    "crawler_timeout": "30",
    "crawler_resolv": "false",
    "feed_refresh_interval": "120",
}


# load the configuration


@@ 52,51 47,53 @@ config = confparser.SafeConfigParser(defaults=DEFAULTS)
config.read(os.path.join(BASE_DIR, "conf/conf.cfg"))


WEBSERVER_HOST = config.get('webserver', 'host')
WEBSERVER_PORT = config.getint('webserver', 'port')
WEBSERVER_SECRET = config.get('webserver', 'secret_key')
WEBSERVER_DEBUG = config.getboolean('webserver', 'debug')
WEBSERVER_HOST = config.get("webserver", "host")
WEBSERVER_PORT = config.getint("webserver", "port")
WEBSERVER_SECRET = config.get("webserver", "secret_key")
WEBSERVER_DEBUG = config.getboolean("webserver", "debug")

CDN_ADDRESS = config.get('cdn', 'cdn_address')
CDN_ADDRESS = config.get("cdn", "cdn_address")

try:
    PLATFORM_URL = config.get('misc', 'platform_url')
    PLATFORM_URL = config.get("misc", "platform_url")
except:
    PLATFORM_URL = "https://www.newspipe.org/"
ADMIN_EMAIL = config.get('misc', 'admin_email')
SELF_REGISTRATION = config.getboolean('misc', 'self_registration')
SECURITY_PASSWORD_SALT = config.get('misc', 'security_password_salt')
ADMIN_EMAIL = config.get("misc", "admin_email")
SELF_REGISTRATION = config.getboolean("misc", "self_registration")
SECURITY_PASSWORD_SALT = config.get("misc", "security_password_salt")
try:
    TOKEN_VALIDITY_PERIOD = config.getint('misc', 'token_validity_period')
    TOKEN_VALIDITY_PERIOD = config.getint("misc", "token_validity_period")
except:
    TOKEN_VALIDITY_PERIOD = int(config.get('misc', 'token_validity_period'))
LOG_PATH = os.path.abspath(config.get('misc', 'log_path'))
LOG_LEVEL = {'debug': logging.DEBUG,
             'info': logging.INFO,
             'warn': logging.WARN,
             'error': logging.ERROR,
             'fatal': logging.FATAL}[config.get('misc', 'log_level')]
    TOKEN_VALIDITY_PERIOD = int(config.get("misc", "token_validity_period"))
LOG_PATH = os.path.abspath(config.get("misc", "log_path"))
LOG_LEVEL = {
    "debug": logging.DEBUG,
    "info": logging.INFO,
    "warn": logging.WARN,
    "error": logging.ERROR,
    "fatal": logging.FATAL,
}[config.get("misc", "log_level")]

SQLALCHEMY_DATABASE_URI = config.get('database', 'database_url')
SQLALCHEMY_DATABASE_URI = config.get("database", "database_url")

CRAWLING_METHOD = config.get('crawler', 'crawling_method')
CRAWLER_USER_AGENT = config.get('crawler', 'user_agent')
DEFAULT_MAX_ERROR = config.getint('crawler', 'default_max_error')
CRAWLING_METHOD = config.get("crawler", "crawling_method")
CRAWLER_USER_AGENT = config.get("crawler", "user_agent")
DEFAULT_MAX_ERROR = config.getint("crawler", "default_max_error")
ERROR_THRESHOLD = int(DEFAULT_MAX_ERROR / 2)
CRAWLER_TIMEOUT = config.get('crawler', 'timeout')
CRAWLER_RESOLV = config.getboolean('crawler', 'resolv')
CRAWLER_TIMEOUT = config.get("crawler", "timeout")
CRAWLER_RESOLV = config.getboolean("crawler", "resolv")
try:
    FEED_REFRESH_INTERVAL = config.getint('crawler', 'feed_refresh_interval')
    FEED_REFRESH_INTERVAL = config.getint("crawler", "feed_refresh_interval")
except:
    FEED_REFRESH_INTERVAL = int(config.get('crawler', 'feed_refresh_interval'))
    FEED_REFRESH_INTERVAL = int(config.get("crawler", "feed_refresh_interval"))

NOTIFICATION_EMAIL = config.get('notification', 'notification_email')
NOTIFICATION_HOST = config.get('notification', 'host')
NOTIFICATION_PORT = config.getint('notification', 'port')
NOTIFICATION_TLS = config.getboolean('notification', 'tls')
NOTIFICATION_SSL = config.getboolean('notification', 'ssl')
NOTIFICATION_USERNAME = config.get('notification', 'username')
NOTIFICATION_PASSWORD = config.get('notification', 'password')
NOTIFICATION_EMAIL = config.get("notification", "notification_email")
NOTIFICATION_HOST = config.get("notification", "host")
NOTIFICATION_PORT = config.getint("notification", "port")
NOTIFICATION_TLS = config.getboolean("notification", "tls")
NOTIFICATION_SSL = config.getboolean("notification", "ssl")
NOTIFICATION_USERNAME = config.get("notification", "username")
NOTIFICATION_PASSWORD = config.get("notification", "password")

CSRF_ENABLED = True
# slow database query threshold (in seconds)

M newspipe/crawler/default_crawler.py => newspipe/crawler/default_crawler.py +37 -39
@@ 40,8 40,7 @@ from web.models import User
from web.controllers import FeedController, ArticleController
from lib.utils import jarr_get
from lib.feed_utils import construct_feed_from, is_parsing_ok
from lib.article_utils import construct_article, extract_id, \
                                    get_article_content
from lib.article_utils import construct_article, extract_id, get_article_content

logger = logging.getLogger(__name__)



@@ 57,12 56,12 @@ async def parse_feed(user, feed):
    up_feed = {}
    articles = []
    resp = None
    #with (await sem):
    # with (await sem):
    try:
        logger.info('Retrieving feed {}'.format(feed.link))
        logger.info("Retrieving feed {}".format(feed.link))
        resp = await jarr_get(feed.link, timeout=5)
    except Exception as e:
        logger.info('Problem when reading feed {}'.format(feed.link))
        logger.info("Problem when reading feed {}".format(feed.link))
        return
    finally:
        if None is resp:


@@ 71,38 70,38 @@ async def parse_feed(user, feed):
            content = io.BytesIO(resp.content)
            parsed_feed = feedparser.parse(content)
        except Exception as e:
            up_feed['last_error'] = str(e)
            up_feed['error_count'] = feed.error_count + 1
            up_feed["last_error"] = str(e)
            up_feed["error_count"] = feed.error_count + 1
            logger.exception("error when parsing feed: " + str(e))
        finally:
            up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal())
            up_feed["last_retrieved"] = datetime.now(dateutil.tz.tzlocal())
            if parsed_feed is None:
                try:
                    FeedController().update({'id': feed.id}, up_feed)
                    FeedController().update({"id": feed.id}, up_feed)
                except Exception as e:
                    logger.exception('something bad here: ' + str(e))
                    logger.exception("something bad here: " + str(e))
                return

    if not is_parsing_ok(parsed_feed):
        up_feed['last_error'] = str(parsed_feed['bozo_exception'])
        up_feed['error_count'] = feed.error_count + 1
        FeedController().update({'id': feed.id}, up_feed)
        up_feed["last_error"] = str(parsed_feed["bozo_exception"])
        up_feed["error_count"] = feed.error_count + 1
        FeedController().update({"id": feed.id}, up_feed)
        return
    if parsed_feed['entries'] != []:
        articles = parsed_feed['entries']
    if parsed_feed["entries"] != []:
        articles = parsed_feed["entries"]

    up_feed['error_count'] = 0
    up_feed['last_error'] = ""
    up_feed["error_count"] = 0
    up_feed["last_error"] = ""

    # Feed information
    try:
        construct_feed_from(feed.link, parsed_feed).update(up_feed)
    except:
         logger.exception('error when constructing feed: {}'.format(feed.link))
    if feed.title and 'title' in up_feed:
        logger.exception("error when constructing feed: {}".format(feed.link))
    if feed.title and "title" in up_feed:
        # do not override the title set by the user
        del up_feed['title']
    FeedController().update({'id': feed.id}, up_feed)
        del up_feed["title"]
    FeedController().update({"id": feed.id}, up_feed)

    return articles



@@ 116,19 115,18 @@ async def insert_articles(queue, nḅ_producers=1):
        if item is None:
            nb_producers_done += 1
            if nb_producers_done == nḅ_producers:
                print('All producers done.')
                print('Process finished.')
                print("All producers done.")
                print("Process finished.")
                break
            continue

        user, feed, articles = item


        if None is articles:
            logger.info('None')
            logger.info("None")
            articles = []

        logger.info('Inserting articles for {}'.format(feed.link))
        logger.info("Inserting articles for {}".format(feed.link))

        art_contr = ArticleController(user.id)
        for article in articles:


@@ 136,9 134,8 @@ async def insert_articles(queue, nḅ_producers=1):

            try:
                existing_article_req = art_contr.read(
                                        user_id=user.id,
                                        feed_id=feed.id,
                                        entry_id=extract_id(article))
                    user_id=user.id, feed_id=feed.id, entry_id=extract_id(article)
                )
            except Exception as e:
                logger.exception("existing_article_req: " + str(e))
                continue


@@ 149,9 146,9 @@ async def insert_articles(queue, nḅ_producers=1):
            # insertion of the new article
            try:
                art_contr.create(**new_article)
                logger.info('New article added: {}'.format(new_article['link']))
                logger.info("New article added: {}".format(new_article["link"]))
            except Exception:
                logger.exception('Error when inserting article in database.')
                logger.exception("Error when inserting article in database.")
                continue




@@ 160,19 157,20 @@ async def retrieve_feed(queue, users, feed_id=None):
    Launch the processus.
    """
    for user in users:
        logger.info('Starting to retrieve feeds for {}'.format(user.nickname))
        logger.info("Starting to retrieve feeds for {}".format(user.nickname))
        filters = {}
        filters['user_id'] = user.id
        filters["user_id"] = user.id
        if feed_id is not None:
            filters['id'] = feed_id
        filters['enabled'] = True
        filters['error_count__lt'] = conf.DEFAULT_MAX_ERROR
        filters['last_retrieved__lt'] = datetime.now() - \
                                    timedelta(minutes=conf.FEED_REFRESH_INTERVAL)
            filters["id"] = feed_id
        filters["enabled"] = True
        filters["error_count__lt"] = conf.DEFAULT_MAX_ERROR
        filters["last_retrieved__lt"] = datetime.now() - timedelta(
            minutes=conf.FEED_REFRESH_INTERVAL
        )
        feeds = FeedController().read(**filters).all()

        if feeds == []:
            logger.info('No feed to retrieve for {}'.format(user.nickname))
            logger.info("No feed to retrieve for {}".format(user.nickname))

        for feed in feeds:
            articles = await parse_feed(user, feed)

M newspipe/lib/article_utils.py => newspipe/lib/article_utils.py +93 -65
@@ 13,69 13,77 @@ import conf
from lib.utils import jarr_get

logger = logging.getLogger(__name__)
PROCESSED_DATE_KEYS = {'published', 'created', 'updated'}
PROCESSED_DATE_KEYS = {"published", "created", "updated"}


def extract_id(entry):
    """ extract a value from an entry that will identify it among the other of
    that feed"""
    return entry.get('entry_id') or entry.get('id') or entry['link']
    return entry.get("entry_id") or entry.get("id") or entry["link"]


async def construct_article(entry, feed, fields=None, fetch=True):
    "Safe method to transform a feedparser entry into an article"
    now = datetime.utcnow()
    article = {}

    def push_in_article(key, value):
        if not fields or key in fields:
            article[key] = value
    push_in_article('feed_id', feed.id)
    push_in_article('user_id', feed.user_id)
    push_in_article('entry_id', extract_id(entry))
    push_in_article('retrieved_date', now)
    if not fields or 'date' in fields:

    push_in_article("feed_id", feed.id)
    push_in_article("user_id", feed.user_id)
    push_in_article("entry_id", extract_id(entry))
    push_in_article("retrieved_date", now)
    if not fields or "date" in fields:
        for date_key in PROCESSED_DATE_KEYS:
            if entry.get(date_key):
                try:
                    article['date'] = dateutil.parser.parse(entry[date_key])\
                            .astimezone(timezone.utc)
                    article["date"] = dateutil.parser.parse(entry[date_key]).astimezone(
                        timezone.utc
                    )
                except Exception as e:
                    logger.exception(e)
                else:
                    break
    push_in_article('content', get_article_content(entry))
    if fields is None or {'link', 'title'}.intersection(fields):
    push_in_article("content", get_article_content(entry))
    if fields is None or {"link", "title"}.intersection(fields):
        link, title = await get_article_details(entry, fetch)
        push_in_article('link', link)
        push_in_article('title', title)
        if 'content' in article:
            #push_in_article('content', clean_urls(article['content'], link))
            push_in_article('content', article['content'])
    push_in_article('tags', {tag.get('term').strip()
                             for tag in entry.get('tags', []) \
                                    if tag and tag.get('term', False)})
        push_in_article("link", link)
        push_in_article("title", title)
        if "content" in article:
            # push_in_article('content', clean_urls(article['content'], link))
            push_in_article("content", article["content"])
    push_in_article(
        "tags",
        {
            tag.get("term").strip()
            for tag in entry.get("tags", [])
            if tag and tag.get("term", False)
        },
    )
    return article


def get_article_content(entry):
    content = ''
    if entry.get('content'):
        content = entry['content'][0]['value']
    elif entry.get('summary'):
        content = entry['summary']
    content = ""
    if entry.get("content"):
        content = entry["content"][0]["value"]
    elif entry.get("summary"):
        content = entry["summary"]
    return content


async def get_article_details(entry, fetch=True):
    article_link = entry.get('link')
    article_title = html.unescape(entry.get('title', ''))
    article_link = entry.get("link")
    article_title = html.unescape(entry.get("title", ""))
    if fetch and conf.CRAWLER_RESOLV and article_link or not article_title:
        try:
            # resolves URL behind proxies (like feedproxy.google.com)
            response = await jarr_get(article_link, timeout=5)
        except MissingSchema:
            split, failed = urlsplit(article_link), False
            for scheme in 'https', 'http':
            for scheme in "https", "http":
                new_link = urlunsplit(SplitResult(scheme, *split[1:]))
                try:
                    response = await jarr_get(new_link, timeout=5)


@@ 86,39 94,44 @@ async def get_article_details(entry, fetch=True):
                article_link = new_link
                break
            if failed:
                return article_link, article_title or 'No title'
                return article_link, article_title or "No title"
        except Exception as error:
            logger.info("Unable to get the real URL of %s. Won't fix "
                        "link or title. Error: %s", article_link, error)
            return article_link, article_title or 'No title'
            logger.info(
                "Unable to get the real URL of %s. Won't fix "
                "link or title. Error: %s",
                article_link,
                error,
            )
            return article_link, article_title or "No title"
        article_link = response.url
        if not article_title:
            bs_parsed = BeautifulSoup(response.content, 'html.parser',
                                      parse_only=SoupStrainer('head'))
            bs_parsed = BeautifulSoup(
                response.content, "html.parser", parse_only=SoupStrainer("head")
            )
            try:
                article_title = bs_parsed.find_all('title')[0].text
                article_title = bs_parsed.find_all("title")[0].text
            except IndexError:  # no title
                pass
    return article_link, article_title or 'No title'
    return article_link, article_title or "No title"


class FiltersAction(Enum):
    READ = 'mark as read'
    LIKED = 'mark as favorite'
    SKIP = 'skipped'
    READ = "mark as read"
    LIKED = "mark as favorite"
    SKIP = "skipped"


class FiltersType(Enum):
    REGEX = 'regex'
    MATCH = 'simple match'
    EXACT_MATCH = 'exact match'
    TAG_MATCH = 'tag match'
    TAG_CONTAINS = 'tag contains'
    REGEX = "regex"
    MATCH = "simple match"
    EXACT_MATCH = "exact match"
    TAG_MATCH = "tag match"
    TAG_CONTAINS = "tag contains"


class FiltersTrigger(Enum):
    MATCH = 'match'
    NO_MATCH = 'no match'
    MATCH = "match"
    NO_MATCH = "no match"


def process_filters(filters, article, only_actions=None):


@@ 129,25 142,30 @@ def process_filters(filters, article, only_actions=None):
    for filter_ in filters:
        match = False
        try:
            pattern = filter_.get('pattern', '')
            filter_type = FiltersType(filter_.get('type'))
            filter_action = FiltersAction(filter_.get('action'))
            filter_trigger = FiltersTrigger(filter_.get('action on'))
            pattern = filter_.get("pattern", "")
            filter_type = FiltersType(filter_.get("type"))
            filter_action = FiltersAction(filter_.get("action"))
            filter_trigger = FiltersTrigger(filter_.get("action on"))
            if filter_type is not FiltersType.REGEX:
                pattern = pattern.lower()
        except ValueError:
            continue
        if filter_action not in only_actions:
            logger.debug('ignoring filter %r' % filter_)
            logger.debug("ignoring filter %r" % filter_)
            continue
        if filter_action in {FiltersType.REGEX, FiltersType.MATCH,
                FiltersType.EXACT_MATCH} and 'title' not in article:
        if (
            filter_action
            in {FiltersType.REGEX, FiltersType.MATCH, FiltersType.EXACT_MATCH}
            and "title" not in article
        ):
            continue
        if filter_action in {FiltersType.TAG_MATCH, FiltersType.TAG_CONTAINS} \
                and 'tags' not in article:
        if (
            filter_action in {FiltersType.TAG_MATCH, FiltersType.TAG_CONTAINS}
            and "tags" not in article
        ):
            continue
        title = article.get('title', '').lower()
        tags = [tag.lower() for tag in article.get('tags', [])]
        title = article.get("title", "").lower()
        tags = [tag.lower() for tag in article.get("tags", [])]
        if filter_type is FiltersType.REGEX:
            match = re.match(pattern, title)
        elif filter_type is FiltersType.MATCH:


@@ 158,8 176,12 @@ def process_filters(filters, article, only_actions=None):
            match = pattern in tags
        elif filter_type is FiltersType.TAG_CONTAINS:
            match = any(pattern in tag for tag in tags)
        take_action = match and filter_trigger is FiltersTrigger.MATCH \
                or not match and filter_trigger is FiltersTrigger.NO_MATCH
        take_action = (
            match
            and filter_trigger is FiltersTrigger.MATCH
            or not match
            and filter_trigger is FiltersTrigger.NO_MATCH
        )

        if not take_action:
            continue


@@ 172,15 194,21 @@ def process_filters(filters, article, only_actions=None):
            skipped = True

    if skipped or read or liked:
        logger.info("%r applied on %r", filter_action.value,
                    article.get('link') or article.get('title'))
        logger.info(
            "%r applied on %r",
            filter_action.value,
            article.get("link") or article.get("title"),
        )
    return skipped, read, liked


def get_skip_and_ids(entry, feed):
    entry_ids = construct_article(entry, feed,
                {'entry_id', 'feed_id', 'user_id'}, fetch=False)
    skipped, _, _ = process_filters(feed.filters,
            construct_article(entry, feed, {'title', 'tags'}, fetch=False),
            {FiltersAction.SKIP})
    entry_ids = construct_article(
        entry, feed, {"entry_id", "feed_id", "user_id"}, fetch=False
    )
    skipped, _, _ = process_filters(
        feed.filters,
        construct_article(entry, feed, {"title", "tags"}, fetch=False),
        {FiltersAction.SKIP},
    )
    return skipped, entry_ids

M newspipe/lib/data.py => newspipe/lib/data.py +96 -67
@@ 1,5 1,5 @@
#! /usr/bin/env python
#-*- coding: utf-8 -*-
# -*- coding: utf-8 -*-

# Newspipe - A Web based news aggregator.
# Copyright (C) 2010-2018  Cédric Bonhomme - https://www.cedricbonhomme.org


@@ 72,18 72,28 @@ def import_opml(nickname, opml_content):
                    link = subscription.xmlUrl
                except:
                    continue
                if None != Feed.query.filter(Feed.user_id == user.id, Feed.link == link).first():
                if (
                    None
                    != Feed.query.filter(
                        Feed.user_id == user.id, Feed.link == link
                    ).first()
                ):
                    continue
                try:
                    site_link = subscription.htmlUrl
                except:
                    site_link = ""
                new_feed = Feed(title=title, description=description,
                                link=link, site_link=site_link,
                                enabled=True)
                new_feed = Feed(
                    title=title,
                    description=description,
                    link=link,
                    site_link=site_link,
                    enabled=True,
                )
                user.feeds.append(new_feed)
                nb += 1
        return nb

    nb = read(subscriptions)
    db.session.commit()
    return nb


@@ 98,40 108,53 @@ def import_json(nickname, json_content):
    nb_feeds, nb_articles = 0, 0
    # Create feeds:
    for feed in json_account:
        if None != Feed.query.filter(Feed.user_id == user.id,
                                    Feed.link == feed["link"]).first():
        if (
            None
            != Feed.query.filter(
                Feed.user_id == user.id, Feed.link == feed["link"]
            ).first()
        ):
            continue
        new_feed = Feed(title=feed["title"],
                        description="",
                        link=feed["link"],
                        site_link=feed["site_link"],
                        created_date=datetime.datetime.
                            fromtimestamp(int(feed["created_date"])),
                        enabled=feed["enabled"])
        new_feed = Feed(
            title=feed["title"],
            description="",
            link=feed["link"],
            site_link=feed["site_link"],
            created_date=datetime.datetime.fromtimestamp(int(feed["created_date"])),
            enabled=feed["enabled"],
        )
        user.feeds.append(new_feed)
        nb_feeds += 1
    db.session.commit()
    # Create articles:
    for feed in json_account:
        user_feed = Feed.query.filter(Feed.user_id == user.id,
                                        Feed.link == feed["link"]).first()
        user_feed = Feed.query.filter(
            Feed.user_id == user.id, Feed.link == feed["link"]
        ).first()
        if None != user_feed:
            for article in feed["articles"]:
                if None == Article.query.filter(Article.user_id == user.id,
                                    Article.feed_id == user_feed.id,
                                    Article.link == article["link"]).first():
                    new_article = Article(entry_id=article["link"],
                                link=article["link"],
                                title=article["title"],
                                content=article["content"],
                                readed=article["readed"],
                                like=article["like"],
                                retrieved_date=datetime.datetime.
                                    fromtimestamp(int(article["retrieved_date"])),
                                date=datetime.datetime.
                                    fromtimestamp(int(article["date"])),
                                user_id=user.id,
                                feed_id=user_feed.id)
                if (
                    None
                    == Article.query.filter(
                        Article.user_id == user.id,
                        Article.feed_id == user_feed.id,
                        Article.link == article["link"],
                    ).first()
                ):
                    new_article = Article(
                        entry_id=article["link"],
                        link=article["link"],
                        title=article["title"],
                        content=article["content"],
                        readed=article["readed"],
                        like=article["like"],
                        retrieved_date=datetime.datetime.fromtimestamp(
                            int(article["retrieved_date"])
                        ),
                        date=datetime.datetime.fromtimestamp(int(article["date"])),
                        user_id=user.id,
                        feed_id=user_feed.id,
                    )
                    user_feed.articles.append(new_article)
                    nb_articles += 1
    db.session.commit()


@@ 144,23 167,28 @@ def export_json(user):
    """
    articles = []
    for feed in user.feeds:
        articles.append({
            "title": feed.title,
            "description": feed.description,
            "link": feed.link,
            "site_link": feed.site_link,
            "enabled": feed.enabled,
            "created_date": feed.created_date.strftime('%s'),
            "articles": [ {
                "title": article.title,
                "link": article.link,
                "content": article.content,
                "readed": article.readed,
                "like": article.like,
                "date": article.date.strftime('%s'),
                "retrieved_date": article.retrieved_date.strftime('%s')
                                                } for article in feed.articles]
        })
        articles.append(
            {
                "title": feed.title,
                "description": feed.description,
                "link": feed.link,
                "site_link": feed.site_link,
                "enabled": feed.enabled,
                "created_date": feed.created_date.strftime("%s"),
                "articles": [
                    {
                        "title": article.title,
                        "link": article.link,
                        "content": article.content,
                        "readed": article.readed,
                        "like": article.like,
                        "date": article.date.strftime("%s"),
                        "retrieved_date": article.retrieved_date.strftime("%s"),
                    }
                    for article in feed.articles
                ],
            }
        )
    return jsonify(articles)




@@ 173,19 201,18 @@ def import_pinboard_json(user, json_content):
    nb_bookmarks = 0
    for bookmark in bookmarks:
        tags = []
        for tag in bookmark['tags'].split(' '):
        for tag in bookmark["tags"].split(" "):
            new_tag = BookmarkTag(text=tag.strip(), user_id=user.id)
            tags.append(new_tag)
        bookmark_attr = {
                    'href': bookmark['href'],
                    'description': bookmark['extended'],
                    'title': bookmark['description'],
                    'shared': [bookmark['shared']=='yes' and True or False][0],
                    'to_read': [bookmark['toread']=='yes' and True or False][0],
                    'time': datetime.datetime.strptime(bookmark['time'],
                                                        '%Y-%m-%dT%H:%M:%SZ'),
                    'tags': tags
                    }
            "href": bookmark["href"],
            "description": bookmark["extended"],
            "title": bookmark["description"],
            "shared": [bookmark["shared"] == "yes" and True or False][0],
            "to_read": [bookmark["toread"] == "yes" and True or False][0],
            "time": datetime.datetime.strptime(bookmark["time"], "%Y-%m-%dT%H:%M:%SZ"),
            "tags": tags,
        }
        new_bookmark = bookmark_contr.create(**bookmark_attr)
        nb_bookmarks += 1
    return nb_bookmarks


@@ 198,13 225,15 @@ def export_bookmarks(user):
    bookmarks = bookmark_contr.read()
    export = []
    for bookmark in bookmarks:
        export.append({
            'href': bookmark.href,
            'description': bookmark.description,
            'title': bookmark.title,
            'shared': 'yes' if bookmark.shared else 'no',
            'toread': 'yes' if bookmark.to_read else 'no',
            'time': bookmark.time.isoformat(),
            'tags': ' '.join(bookmark.tags_proxy)
        })
        export.append(
            {
                "href": bookmark.href,
                "description": bookmark.description,
                "title": bookmark.title,
                "shared": "yes" if bookmark.shared else "no",
                "toread": "yes" if bookmark.to_read else "no",
                "time": bookmark.time.isoformat(),
                "tags": " ".join(bookmark.tags_proxy),
            }
        )
    return jsonify(export)

M newspipe/lib/feed_utils.py => newspipe/lib/feed_utils.py +62 -53
@@ 10,12 10,17 @@ from lib.utils import try_keys, try_get_icon_url, rebuild_url

logger = logging.getLogger(__name__)
logging.captureWarnings(True)
ACCEPTED_MIMETYPES = ('application/rss+xml', 'application/rdf+xml',
                      'application/atom+xml', 'application/xml', 'text/xml')
ACCEPTED_MIMETYPES = (
    "application/rss+xml",
    "application/rdf+xml",
    "application/atom+xml",
    "application/xml",
    "text/xml",
)


def is_parsing_ok(parsed_feed):
    return parsed_feed['entries'] or not parsed_feed['bozo']
    return parsed_feed["entries"] or not parsed_feed["bozo"]


def escape_keys(*keys):


@@ 24,66 29,71 @@ def escape_keys(*keys):
            result = func(*args, **kwargs)
            for key in keys:
                if key in result:
                    result[key] = html.unescape(result[key] or '')
                    result[key] = html.unescape(result[key] or "")
            return result

        return metawrapper

    return wrapper


@escape_keys('title', 'description')
@escape_keys("title", "description")
def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True):
    requests_kwargs = {'headers': {'User-Agent': CRAWLER_USER_AGENT},
                        'verify': False}
    requests_kwargs = {"headers": {"User-Agent": CRAWLER_USER_AGENT}, "verify": False}
    if url is None and fp_parsed is not None:
        url = fp_parsed.get('url')
        url = fp_parsed.get("url")
    if url is not None and fp_parsed is None:
        try:
            response = requests.get(url, **requests_kwargs)
            fp_parsed = feedparser.parse(response.content,
                                         request_headers=response.headers)
            fp_parsed = feedparser.parse(
                response.content, request_headers=response.headers
            )
        except Exception:
            logger.exception('failed to retrieve that url')
            fp_parsed = {'bozo': True}
            logger.exception("failed to retrieve that url")
            fp_parsed = {"bozo": True}
    assert url is not None and fp_parsed is not None
    feed = feed or {}
    feed_split = urllib.parse.urlsplit(url)
    site_split = None
    if is_parsing_ok(fp_parsed):
        feed['link'] = url
        feed['site_link'] = try_keys(fp_parsed['feed'], 'href', 'link')
        feed['title'] = fp_parsed['feed'].get('title')
        feed['description'] = try_keys(fp_parsed['feed'], 'subtitle', 'title')
        feed['icon_url'] = try_keys(fp_parsed['feed'], 'icon')
        feed["link"] = url
        feed["site_link"] = try_keys(fp_parsed["feed"], "href", "link")
        feed["title"] = fp_parsed["feed"].get("title")
        feed["description"] = try_keys(fp_parsed["feed"], "subtitle", "title")
        feed["icon_url"] = try_keys(fp_parsed["feed"], "icon")
    else:
        feed['site_link'] = url

    if feed.get('site_link'):
        feed['site_link'] = rebuild_url(feed['site_link'], feed_split)
        site_split = urllib.parse.urlsplit(feed['site_link'])

    if feed.get('icon_url'):
        feed['icon_url'] = try_get_icon_url(
                feed['icon_url'], site_split, feed_split)
        if feed['icon_url'] is None:
            del feed['icon_url']

    if not feed.get('site_link') or not query_site \
            or all(bool(feed.get(k)) for k in ('link', 'title', 'icon_url')):
        feed["site_link"] = url

    if feed.get("site_link"):
        feed["site_link"] = rebuild_url(feed["site_link"], feed_split)
        site_split = urllib.parse.urlsplit(feed["site_link"])

    if feed.get("icon_url"):
        feed["icon_url"] = try_get_icon_url(feed["icon_url"], site_split, feed_split)
        if feed["icon_url"] is None:
            del feed["icon_url"]

    if (
        not feed.get("site_link")
        or not query_site
        or all(bool(feed.get(k)) for k in ("link", "title", "icon_url"))
    ):
        return feed

    try:
        response = requests.get(feed['site_link'], **requests_kwargs)
        response = requests.get(feed["site_link"], **requests_kwargs)
    except requests.exceptions.InvalidSchema as e:
        return feed
    except:
        logger.exception('failed to retrieve %r', feed['site_link'])
        logger.exception("failed to retrieve %r", feed["site_link"])
        return feed
    bs_parsed = BeautifulSoup(response.content, 'html.parser',
                              parse_only=SoupStrainer('head'))
    bs_parsed = BeautifulSoup(
        response.content, "html.parser", parse_only=SoupStrainer("head")
    )

    if not feed.get('title'):
    if not feed.get("title"):
        try:
            feed['title'] = bs_parsed.find_all('title')[0].text
            feed["title"] = bs_parsed.find_all("title")[0].text
        except Exception:
            pass



@@ 95,31 105,30 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True):
                if not all(val in elem.attrs[key] for val in vals):
                    return False
            return True

        return wrapper

    if not feed.get('icon_url'):
        icons = bs_parsed.find_all(check_keys(rel=['icon', 'shortcut']))
    if not feed.get("icon_url"):
        icons = bs_parsed.find_all(check_keys(rel=["icon", "shortcut"]))
        if not len(icons):
            icons = bs_parsed.find_all(check_keys(rel=['icon']))
            icons = bs_parsed.find_all(check_keys(rel=["icon"]))
        if len(icons) >= 1:
            for icon in icons:
                feed['icon_url'] = try_get_icon_url(icon.attrs['href'],
                                                    site_split, feed_split)
                if feed['icon_url'] is not None:
                feed["icon_url"] = try_get_icon_url(
                    icon.attrs["href"], site_split, feed_split
                )
                if feed["icon_url"] is not None:
                    break

        if feed.get('icon_url') is None:
            feed['icon_url'] = try_get_icon_url('/favicon.ico',
                                                site_split, feed_split)
        if 'icon_url' in feed and feed['icon_url'] is None:
            del feed['icon_url']
        if feed.get("icon_url") is None:
            feed["icon_url"] = try_get_icon_url("/favicon.ico", site_split, feed_split)
        if "icon_url" in feed and feed["icon_url"] is None:
            del feed["icon_url"]

    if not feed.get('link'):
    if not feed.get("link"):
        for type_ in ACCEPTED_MIMETYPES:
            alternates = bs_parsed.find_all(check_keys(
                    rel=['alternate'], type=[type_]))
            alternates = bs_parsed.find_all(check_keys(rel=["alternate"], type=[type_]))
            if len(alternates) >= 1:
                feed['link'] = rebuild_url(alternates[0].attrs['href'],
                                           feed_split)
                feed["link"] = rebuild_url(alternates[0].attrs["href"], feed_split)
                break
    return feed

M newspipe/lib/misc_utils.py => newspipe/lib/misc_utils.py +43 -30
@@ 1,5 1,5 @@
#! /usr/bin/env python
#-*- coding: utf-8 -*-
# -*- coding: utf-8 -*-

# Newspipe - A Web based news aggregator.
# Copyright (C) 2010-2018  Cédric Bonhomme - https://www.cedricbonhomme.org


@@ 36,6 36,7 @@ import operator
import urllib
import subprocess
import sqlalchemy

try:
    from urlparse import urlparse, parse_qs, urlunparse
except:


@@ 50,7 51,7 @@ from lib.utils import clear_string

logger = logging.getLogger(__name__)

ALLOWED_EXTENSIONS = set(['xml', 'opml', 'json'])
ALLOWED_EXTENSIONS = set(["xml", "opml", "json"])


def is_safe_url(target):


@@ 59,15 60,14 @@ def is_safe_url(target):
    """
    ref_url = urlparse(request.host_url)
    test_url = urlparse(urljoin(request.host_url, target))
    return test_url.scheme in ('http', 'https') and \
           ref_url.netloc == test_url.netloc
    return test_url.scheme in ("http", "https") and ref_url.netloc == test_url.netloc


def get_redirect_target():
    """
    Looks at various hints to find the redirect target.
    """
    for target in request.args.get('next'), request.referrer:
    for target in request.args.get("next"), request.referrer:
        if not target:
            continue
        if is_safe_url(target):


@@ 78,8 78,7 @@ def allowed_file(filename):
    """
    Check if the uploaded file is allowed.
    """
    return '.' in filename and \
            filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS
    return "." in filename and filename.rsplit(".", 1)[1] in ALLOWED_EXTENSIONS


@contextmanager


@@ 100,10 99,14 @@ def fetch(id, feed_id=None):
    Fetch the feeds in a new processus.
    The default crawler ("asyncio") is launched with the manager.
    """
    cmd = [sys.executable, conf.BASE_DIR + '/manager.py', 'fetch_asyncio',
           '--user_id='+str(id)]
    cmd = [
        sys.executable,
        conf.BASE_DIR + "/manager.py",
        "fetch_asyncio",
        "--user_id=" + str(id),
    ]
    if feed_id:
        cmd.append('--feed_id='+str(feed_id))
        cmd.append("--feed_id=" + str(feed_id))
    return subprocess.Popen(cmd, stdout=subprocess.PIPE)




@@ 114,9 117,11 @@ def history(user_id, year=None, month=None):
    articles_counter = Counter()
    articles = ArticleController(user_id).read()
    if None != year:
        articles = articles.filter(sqlalchemy.extract('year', 'Article.date') == year)
        articles = articles.filter(sqlalchemy.extract("year", "Article.date") == year)
        if None != month:
            articles = articles.filter(sqlalchemy.extract('month', 'Article.date') == month)
            articles = articles.filter(
                sqlalchemy.extract("month", "Article.date") == month
            )
    for article in articles.all():
        if None != year:
            articles_counter[article.date.month] += 1


@@ 131,24 136,26 @@ def clean_url(url):
    """
    parsed_url = urlparse(url)
    qd = parse_qs(parsed_url.query, keep_blank_values=True)
    filtered = dict((k, v) for k, v in qd.items()
                                        if not k.startswith('utm_'))
    return urlunparse([
        parsed_url.scheme,
        parsed_url.netloc,
        urllib.parse.quote(urllib.parse.unquote(parsed_url.path)),
        parsed_url.params,
        urllib.parse.urlencode(filtered, doseq=True),
        parsed_url.fragment
    ]).rstrip('=')
    filtered = dict((k, v) for k, v in qd.items() if not k.startswith("utm_"))
    return urlunparse(
        [
            parsed_url.scheme,
            parsed_url.netloc,
            urllib.parse.quote(urllib.parse.unquote(parsed_url.path)),
            parsed_url.params,
            urllib.parse.urlencode(filtered, doseq=True),
            parsed_url.fragment,
        ]
    ).rstrip("=")


def load_stop_words():
    """
    Load the stop words and return them in a list.
    """
    stop_words_lists = glob.glob(os.path.join(conf.BASE_DIR,
                                                'web/var/stop_words/*.txt'))
    stop_words_lists = glob.glob(
        os.path.join(conf.BASE_DIR, "web/var/stop_words/*.txt")
    )
    stop_words = []

    for stop_wods_list in stop_words_lists:


@@ 166,11 173,13 @@ def top_words(articles, n=10, size=5):
    """
    stop_words = load_stop_words()
    words = Counter()
    wordre = re.compile(r'\b\w{%s,}\b' % size, re.I)
    wordre = re.compile(r"\b\w{%s,}\b" % size, re.I)
    for article in articles:
        for word in [elem.lower() for elem in
                wordre.findall(clear_string(article.content)) \
                if elem.lower() not in stop_words]:
        for word in [
            elem.lower()
            for elem in wordre.findall(clear_string(article.content))
            if elem.lower() not in stop_words
        ]:
            words[word] += 1
    return words.most_common(n)



@@ 181,5 190,9 @@ def tag_cloud(tags):
    """
    tags.sort(key=operator.itemgetter(0))
    max_tag = max([tag[1] for tag in tags])
    return '\n'.join([('<font size=%d>%s</font>' % \
        (min(1 + count * 7 / max_tag, 7), word)) for (word, count) in tags])
    return "\n".join(
        [
            ("<font size=%d>%s</font>" % (min(1 + count * 7 / max_tag, 7), word))
            for (word, count) in tags
        ]
    )

M newspipe/lib/utils.py => newspipe/lib/utils.py +32 -21
@@ 11,18 11,20 @@ import conf
logger = logging.getLogger(__name__)


def default_handler(obj, role='admin'):
def default_handler(obj, role="admin"):
    """JSON handler for default query formatting"""
    if hasattr(obj, 'isoformat'):
    if hasattr(obj, "isoformat"):
        return obj.isoformat()
    if hasattr(obj, 'dump'):
    if hasattr(obj, "dump"):
        return obj.dump(role=role)
    if isinstance(obj, (set, frozenset, types.GeneratorType)):
        return list(obj)
    if isinstance(obj, BaseException):
        return str(obj)
    raise TypeError("Object of type %s with value of %r "
                    "is not JSON serializable" % (type(obj), obj))
    raise TypeError(
        "Object of type %s with value of %r "
        "is not JSON serializable" % (type(obj), obj)
    )


def try_keys(dico, *keys):


@@ 37,9 39,12 @@ def rebuild_url(url, base_split):
    if split.scheme and split.netloc:
        return url  # url is fine
    new_split = urllib.parse.SplitResult(
            scheme=split.scheme or base_split.scheme,
            netloc=split.netloc or base_split.netloc,
            path=split.path, query='', fragment='')
        scheme=split.scheme or base_split.scheme,
        netloc=split.netloc or base_split.netloc,
        path=split.path,
        query="",
        fragment="",
    )
    return urllib.parse.urlunsplit(new_split)




@@ 52,19 57,22 @@ def try_get_icon_url(url, *splits):
        # if html in content-type, we assume it's a fancy 404 page
        try:
            response = jarr_get(rb_url)
            content_type = response.headers.get('content-type', '')
            content_type = response.headers.get("content-type", "")
        except Exception:
            pass
        else:
            if response is not None and response.ok \
                    and 'html' not in content_type and response.content:
            if (
                response is not None
                and response.ok
                and "html" not in content_type
                and response.content
            ):
                return response.url
    return None


def to_hash(text):
    return md5(text.encode('utf8') if hasattr(text, 'encode') else text)\
            .hexdigest()
    return md5(text.encode("utf8") if hasattr(text, "encode") else text).hexdigest()


def clear_string(data):


@@ 72,18 80,21 @@ def clear_string(data):
    Clear a string by removing HTML tags, HTML special caracters
    and consecutive white spaces (more that one).
    """
    p = re.compile('<[^>]+>')  # HTML tags
    q = re.compile('\s')  # consecutive white spaces
    return p.sub('', q.sub(' ', data))
    p = re.compile("<[^>]+>")  # HTML tags
    q = re.compile("\s")  # consecutive white spaces
    return p.sub("", q.sub(" ", data))


def redirect_url(default='home'):
    return request.args.get('next') or request.referrer or url_for(default)
def redirect_url(default="home"):
    return request.args.get("next") or request.referrer or url_for(default)


async def jarr_get(url, **kwargs):
    request_kwargs = {'verify': False, 'allow_redirects': True,
                      'timeout': conf.CRAWLER_TIMEOUT,
                      'headers': {'User-Agent': conf.CRAWLER_USER_AGENT}}
    request_kwargs = {
        "verify": False,
        "allow_redirects": True,
        "timeout": conf.CRAWLER_TIMEOUT,
        "headers": {"User-Agent": conf.CRAWLER_USER_AGENT},
    }
    request_kwargs.update(kwargs)
    return requests.get(url, **request_kwargs)

M newspipe/manager.py => newspipe/manager.py +27 -19
@@ 12,12 12,12 @@ from flask_migrate import Migrate, MigrateCommand
import web.models
from web.controllers import UserController

logger = logging.getLogger('manager')
logger = logging.getLogger("manager")

Migrate(application, db)

manager = Manager(application)
manager.add_command('db', MigrateCommand)
manager.add_command("db", MigrateCommand)


@manager.command


@@ 30,23 30,32 @@ def db_empty():
@manager.command
def db_create():
    "Will create the database from conf parameters."
    admin = {'is_admin': True, 'is_api': True, 'is_active': True,
             'nickname': 'admin',
             'pwdhash': generate_password_hash(
                            os.environ.get("ADMIN_PASSWORD", "password"))}
    admin = {
        "is_admin": True,
        "is_api": True,
        "is_active": True,
        "nickname": "admin",
        "pwdhash": generate_password_hash(os.environ.get("ADMIN_PASSWORD", "password")),
    }
    with application.app_context():
        db.create_all()
        UserController(ignore_context=True).create(**admin)


@manager.command
def create_admin(nickname, password):
    "Will create an admin user."
    admin = {'is_admin': True, 'is_api': True, 'is_active': True,
             'nickname': nickname,
             'pwdhash': generate_password_hash(password)}
    admin = {
        "is_admin": True,
        "is_api": True,
        "is_active": True,
        "nickname": nickname,
        "pwdhash": generate_password_hash(password),
    }
    with application.app_context():
        UserController(ignore_context=True).create(**admin)


@manager.command
def fetch_asyncio(user_id=None, feed_id=None):
    "Crawl the feeds with asyncio."


@@ 54,33 63,32 @@ def fetch_asyncio(user_id=None, feed_id=None):

    with application.app_context():
        from crawler import default_crawler

        filters = {}
        filters['is_active'] = True
        filters['automatic_crawling'] = True
        filters["is_active"] = True
        filters["automatic_crawling"] = True
        if None is not user_id:
            filters['id'] = user_id
            filters["id"] = user_id
        users = UserController().read(**filters).all()

        try:
            feed_id = int(feed_id)
        except:
            feed_id = None
            
        

        loop = asyncio.get_event_loop()
        queue = asyncio.Queue(maxsize=3, loop=loop)
        

        producer_coro = default_crawler.retrieve_feed(queue, users, feed_id)
        consumer_coro = default_crawler.insert_articles(queue, 1)

        logger.info('Starting crawler.')
        logger.info("Starting crawler.")
        start = datetime.now()
        loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro))
        end = datetime.now()
        loop.close()
        logger.info('Crawler finished in {} seconds.' \
                        .format((end - start).seconds))
        logger.info("Crawler finished in {} seconds.".format((end - start).seconds))


if __name__ == '__main__':
if __name__ == "__main__":
    manager.run()

M newspipe/notifications/emails.py => newspipe/notifications/emails.py +13 -9
@@ 36,31 36,33 @@ def send_async_email(mfrom, mto, msg):
        s = smtplib.SMTP(conf.NOTIFICATION_HOST)
        s.login(conf.NOTIFICATION_USERNAME, conf.NOTIFICATION_PASSWORD)
    except Exception:
        logger.exception('send_async_email raised:')
        logger.exception("send_async_email raised:")
    else:
        s.sendmail(mfrom, mto, msg.as_string())
        s.quit()


def send(*args, **kwargs):
    """
    This functions enables to send email via different method.
    """
    send_smtp(**kwargs)


def send_smtp(to="", bcc="", subject="", plaintext="", html=""):
    """
    Send an email.
    """
    # Create message container - the correct MIME type is multipart/alternative.
    msg = MIMEMultipart('alternative')
    msg['Subject'] = subject
    msg['From'] = conf.NOTIFICATION_EMAIL
    msg['To'] = to
    msg['BCC'] = bcc
    msg = MIMEMultipart("alternative")
    msg["Subject"] = subject
    msg["From"] = conf.NOTIFICATION_EMAIL
    msg["To"] = to
    msg["BCC"] = bcc

    # Record the MIME types of both parts - text/plain and text/html.
    part1 = MIMEText(plaintext, 'plain', 'utf-8')
    part2 = MIMEText(html, 'html', 'utf-8')
    part1 = MIMEText(plaintext, "plain", "utf-8")
    part2 = MIMEText(html, "html", "utf-8")

    # Attach parts into message container.
    # According to RFC 2046, the last part of a multipart message, in this case


@@ 74,5 76,7 @@ def send_smtp(to="", bcc="", subject="", plaintext="", html=""):
    except Exception:
        logger.exception("send_smtp raised:")
    else:
        s.sendmail(conf.NOTIFICATION_EMAIL, msg['To'] + ", " + msg['BCC'], msg.as_string())
        s.sendmail(
            conf.NOTIFICATION_EMAIL, msg["To"] + ", " + msg["BCC"], msg.as_string()
        )
        s.quit()

M newspipe/notifications/notifications.py => newspipe/notifications/notifications.py +24 -13
@@ 31,23 31,34 @@ def new_account_notification(user, email):
    Account creation notification.
    """
    token = generate_confirmation_token(user.nickname)
    expire_time = datetime.datetime.now() + \
                    datetime.timedelta(seconds=conf.TOKEN_VALIDITY_PERIOD)
    expire_time = datetime.datetime.now() + datetime.timedelta(
        seconds=conf.TOKEN_VALIDITY_PERIOD
    )

    plaintext = render_template('emails/account_activation.txt',
                                    user=user, platform_url=conf.PLATFORM_URL,
                                    token=token,
                                    expire_time=expire_time)
    plaintext = render_template(
        "emails/account_activation.txt",
        user=user,
        platform_url=conf.PLATFORM_URL,
        token=token,
        expire_time=expire_time,
    )

    emails.send(
        to=email,
        bcc=conf.NOTIFICATION_EMAIL,
        subject="[Newspipe] Account creation",
        plaintext=plaintext,
    )

    emails.send(to=email, bcc=conf.NOTIFICATION_EMAIL,
                subject="[Newspipe] Account creation", plaintext=plaintext)

def new_password_notification(user, password):
    """
    New password notification.
    """
    plaintext = render_template('emails/new_password.txt',
                                    user=user, password=password)
    emails.send(to=user.email,
                bcc=conf.NOTIFICATION_EMAIL,
                subject="[Newspipe] New password", plaintext=plaintext)
    plaintext = render_template("emails/new_password.txt", user=user, password=password)
    emails.send(
        to=user.email,
        bcc=conf.NOTIFICATION_EMAIL,
        subject="[Newspipe] New password",
        plaintext=plaintext,
    )

M newspipe/runserver.py => newspipe/runserver.py +11 -8
@@ 29,9 29,11 @@ babel = Babel(application)
# Jinja filters
def month_name(month_number):
    return calendar.month_name[month_number]
application.jinja_env.filters['month_name'] = month_name
application.jinja_env.filters['datetime'] = format_datetime
application.jinja_env.globals['conf'] = conf


application.jinja_env.filters["month_name"] = month_name
application.jinja_env.filters["datetime"] = format_datetime
application.jinja_env.globals["conf"] = conf

# Views
from flask_restful import Api


@@ 39,10 41,11 @@ from flask import g

with application.app_context():
    populate_g()
    g.api = Api(application, prefix='/api/v2.0')
    g.api = Api(application, prefix="/api/v2.0")
    g.babel = babel

    from web import views

    application.register_blueprint(views.articles_bp)
    application.register_blueprint(views.article_bp)
    application.register_blueprint(views.feeds_bp)


@@ 57,7 60,7 @@ with application.app_context():
    application.register_blueprint(views.bookmark_bp)


if __name__ == '__main__':
    application.run(host=conf.WEBSERVER_HOST,
                    port=conf.WEBSERVER_PORT,
                    debug=conf.WEBSERVER_DEBUG)
if __name__ == "__main__":
    application.run(
        host=conf.WEBSERVER_HOST, port=conf.WEBSERVER_PORT, debug=conf.WEBSERVER_DEBUG
    )

M newspipe/web/controllers/__init__.py => newspipe/web/controllers/__init__.py +9 -3
@@ 7,6 7,12 @@ from .bookmark import BookmarkController
from .tag import BookmarkTagController


__all__ = ['FeedController', 'CategoryController', 'ArticleController',
           'UserController', 'IconController', 'BookmarkController',
           'BookmarkTagController']
__all__ = [
    "FeedController",
    "CategoryController",
    "ArticleController",
    "UserController",
    "IconController",
    "BookmarkController",
    "BookmarkTagController",
]

M newspipe/web/controllers/abstract.py => newspipe/web/controllers/abstract.py +53 -37
@@ 11,7 11,7 @@ logger = logging.getLogger(__name__)

class AbstractController:
    _db_cls = None  # reference to the database class
    _user_id_key = 'user_id'
    _user_id_key = "user_id"

    def __init__(self, user_id=None, ignore_context=False):
        """User id is a right management mechanism that should be used to


@@ 36,25 36,25 @@ class AbstractController:
        """
        db_filters = set()
        for key, value in filters.items():
            if key == '__or__':
            if key == "__or__":
                db_filters.add(or_(*self._to_filters(**value)))
            elif key.endswith('__gt'):
            elif key.endswith("__gt"):
                db_filters.add(getattr(self._db_cls, key[:-4]) > value)
            elif key.endswith('__lt'):
            elif key.endswith("__lt"):
                db_filters.add(getattr(self._db_cls, key[:-4]) < value)
            elif key.endswith('__ge'):
            elif key.endswith("__ge"):
                db_filters.add(getattr(self._db_cls, key[:-4]) >= value)
            elif key.endswith('__le'):
            elif key.endswith("__le"):
                db_filters.add(getattr(self._db_cls, key[:-4]) <= value)
            elif key.endswith('__ne'):
            elif key.endswith("__ne"):
                db_filters.add(getattr(self._db_cls, key[:-4]) != value)
            elif key.endswith('__in'):
            elif key.endswith("__in"):
                db_filters.add(getattr(self._db_cls, key[:-4]).in_(value))
            elif key.endswith('__contains'):
            elif key.endswith("__contains"):
                db_filters.add(getattr(self._db_cls, key[:-10]).contains(value))
            elif key.endswith('__like'):
            elif key.endswith("__like"):
                db_filters.add(getattr(self._db_cls, key[:-6]).like(value))
            elif key.endswith('__ilike'):
            elif key.endswith("__ilike"):
                db_filters.add(getattr(self._db_cls, key[:-7]).ilike(value))
            else:
                db_filters.add(getattr(self._db_cls, key) == value)


@@ 66,8 66,11 @@ class AbstractController:
        dependent) and the user is not an admin and the filters doesn't already
        contains a filter for that user.
        """
        if self._user_id_key is not None and self.user_id \
                and filters.get(self._user_id_key) != self.user_id:
        if (
            self._user_id_key is not None
            and self.user_id
            and filters.get(self._user_id_key) != self.user_id
        ):
            filters[self._user_id_key] = self.user_id
        return self._db_cls.query.filter(*self._to_filters(**filters))



@@ 76,20 79,27 @@ class AbstractController:
        obj = self._get(**filters).first()

        if obj and not self._has_right_on(obj):
            raise Forbidden({'message': 'No authorized to access %r (%r)'
                                % (self._db_cls.__class__.__name__, filters)})
            raise Forbidden(
                {
                    "message": "No authorized to access %r (%r)"
                    % (self._db_cls.__class__.__name__, filters)
                }
            )
        if not obj:
            raise NotFound({'message': 'No %r (%r)'
                                % (self._db_cls.__class__.__name__, filters)})
            raise NotFound(
                {"message": "No %r (%r)" % (self._db_cls.__class__.__name__, filters)}
            )
        return obj

    def create(self, **attrs):
        assert attrs, "attributes to update must not be empty"
        if self._user_id_key is not None and self._user_id_key not in attrs:
            attrs[self._user_id_key] = self.user_id
        assert self._user_id_key is None or self._user_id_key in attrs \
                or self.user_id is None, \
                "You must provide user_id one way or another"
        assert (
            self._user_id_key is None
            or self._user_id_key in attrs
            or self.user_id is None
        ), "You must provide user_id one way or another"

        obj = self._db_cls(**attrs)
        db.session.add(obj)


@@ 123,39 133,45 @@ class AbstractController:
        # user_id == None is like being admin
        if self._user_id_key is None:
            return True
        return self.user_id is None \
                or getattr(obj, self._user_id_key, None) == self.user_id
        return (
            self.user_id is None
            or getattr(obj, self._user_id_key, None) == self.user_id
        )

    def _count_by(self, elem_to_group_by, filters):
        if self.user_id:
            filters['user_id'] = self.user_id
        return dict(db.session.query(elem_to_group_by, func.count('id'))
                              .filter(*self._to_filters(**filters))
                              .group_by(elem_to_group_by).all())
            filters["user_id"] = self.user_id
        return dict(
            db.session.query(elem_to_group_by, func.count("id"))
            .filter(*self._to_filters(**filters))
            .group_by(elem_to_group_by)
            .all()
        )

    @classmethod
    def _get_attrs_desc(cls, role, right=None):
        result = defaultdict(dict)
        if role == 'admin':
        if role == "admin":
            columns = cls._db_cls.__table__.columns.keys()
        else:
            assert role in {'base', 'api'}, 'unknown role %r' % role
            assert right in {'read', 'write'}, \
                    "right must be 'read' or 'write' with role %r" % role
            columns = getattr(cls._db_cls, 'fields_%s_%s' % (role, right))()
            assert role in {"base", "api"}, "unknown role %r" % role
            assert right in {"read", "write"}, (
                "right must be 'read' or 'write' with role %r" % role
            )
            columns = getattr(cls._db_cls, "fields_%s_%s" % (role, right))()
        for column in columns:
            result[column] = {}
            db_col = getattr(cls._db_cls, column).property.columns[0]
            try:
                result[column]['type'] = db_col.type.python_type
                result[column]["type"] = db_col.type.python_type
            except NotImplementedError:
                if db_col.default:
                    result[column]['type'] = db_col.default.arg.__class__
                    result[column]["type"] = db_col.default.arg.__class__
            if column not in result:
                continue
            if issubclass(result[column]['type'], datetime):
                result[column]['default'] = datetime.utcnow()
                result[column]['type'] = lambda x: dateutil.parser.parse(x)
            if issubclass(result[column]["type"], datetime):
                result[column]["default"] = datetime.utcnow()
                result[column]["type"] = lambda x: dateutil.parser.parse(x)
            elif db_col.default:
                result[column]['default'] = db_col.default.arg
                result[column]["default"] = db_col.default.arg
        return result

M newspipe/web/controllers/article.py => newspipe/web/controllers/article.py +41 -24
@@ 30,19 30,24 @@ class ArticleController(AbstractController):
        return self._count_by(Article.feed_id, filters)

    def count_by_user_id(self, **filters):
        return dict(db.session.query(Article.user_id, func.count(Article.id))
                              .filter(*self._to_filters(**filters))
                              .group_by(Article.user_id).all())
        return dict(
            db.session.query(Article.user_id, func.count(Article.id))
            .filter(*self._to_filters(**filters))
            .group_by(Article.user_id)
            .all()
        )

    def create(self, **attrs):
        # handling special denorm for article rights
        assert 'feed_id' in attrs, "must provide feed_id when creating article"
        feed = FeedController(
                attrs.get('user_id', self.user_id)).get(id=attrs['feed_id'])
        if 'user_id' in attrs:
            assert feed.user_id == attrs['user_id'] or self.user_id is None, \
                    "no right on feed %r" % feed.id
        attrs['user_id'], attrs['category_id'] = feed.user_id, feed.category_id
        assert "feed_id" in attrs, "must provide feed_id when creating article"
        feed = FeedController(attrs.get("user_id", self.user_id)).get(
            id=attrs["feed_id"]
        )
        if "user_id" in attrs:
            assert feed.user_id == attrs["user_id"] or self.user_id is None, (
                "no right on feed %r" % feed.id
            )
        attrs["user_id"], attrs["category_id"] = feed.user_id, feed.category_id

        skipped, read, liked = process_filters(feed.filters, attrs)
        if skipped:


@@ 51,15 56,16 @@ class ArticleController(AbstractController):
        return article

    def update(self, filters, attrs):
        user_id = attrs.get('user_id', self.user_id)
        if 'feed_id' in attrs:
            feed = FeedController().get(id=attrs['feed_id'])
        user_id = attrs.get("user_id", self.user_id)
        if "feed_id" in attrs:
            feed = FeedController().get(id=attrs["feed_id"])
            assert feed.user_id == user_id, "no right on feed %r" % feed.id
            attrs['category_id'] = feed.category_id
        if attrs.get('category_id'):
            cat = CategoryController().get(id=attrs['category_id'])
            assert self.user_id is None or cat.user_id == user_id, \
                    "no right on cat %r" % cat.id
            attrs["category_id"] = feed.category_id
        if attrs.get("category_id"):
            cat = CategoryController().get(id=attrs["category_id"])
            assert self.user_id is None or cat.user_id == user_id, (
                "no right on cat %r" % cat.id
            )
        return super().update(filters, attrs)

    def get_history(self, year=None, month=None):


@@ 69,11 75,11 @@ class ArticleController(AbstractController):
        articles_counter = Counter()
        articles = self.read()
        if year is not None:
            articles = articles.filter(
                    sqlalchemy.extract('year', Article.date) == year)
            articles = articles.filter(sqlalchemy.extract("year", Article.date) == year)
            if month is not None:
                articles = articles.filter(
                        sqlalchemy.extract('month', Article.date) == month)
                    sqlalchemy.extract("month", Article.date) == month
                )
        for article in articles.all():
            if year is not None:
                articles_counter[article.date.month] += 1


@@ 82,6 88,17 @@ class ArticleController(AbstractController):
        return articles_counter, articles

    def read_light(self, **filters):
        return super().read(**filters).with_entities(Article.id, Article.title,
                Article.readed, Article.like, Article.feed_id, Article.date,
                Article.category_id).order_by(Article.date.desc())
        return (
            super()
            .read(**filters)
            .with_entities(
                Article.id,
                Article.title,
                Article.readed,
                Article.like,
                Article.feed_id,
                Article.date,
                Article.category_id,
            )
            .order_by(Article.date.desc())
        )

M newspipe/web/controllers/bookmark.py => newspipe/web/controllers/bookmark.py +13 -10
@@ 17,16 17,19 @@ class BookmarkController(AbstractController):
        return self._count_by(Bookmark.href, filters)

    def update(self, filters, attrs):
        BookmarkTagController(self.user_id) \
                                .read(**{'bookmark_id': filters["id"]}) \
                                .delete()
        BookmarkTagController(self.user_id).read(
            **{"bookmark_id": filters["id"]}
        ).delete()

        for tag in attrs['tags']:
        for tag in attrs["tags"]:
            BookmarkTagController(self.user_id).create(
                                            **{'text': tag.text,
                                                'id': tag.id,
                                                'bookmark_id': tag.bookmark_id,
                                                'user_id': tag.user_id})

        del attrs['tags']
                **{
                    "text": tag.text,
                    "id": tag.id,
                    "bookmark_id": tag.bookmark_id,
                    "user_id": tag.user_id,
                }
            )

        del attrs["tags"]
        return super().update(filters, attrs)

M newspipe/web/controllers/category.py => newspipe/web/controllers/category.py +3 -2
@@ 7,6 7,7 @@ class CategoryController(AbstractController):
    _db_cls = Category

    def delete(self, obj_id):
        FeedController(self.user_id).update({'category_id': obj_id},
                                            {'category_id': None})
        FeedController(self.user_id).update(
            {"category_id": obj_id}, {"category_id": None}
        )
        return super().delete(obj_id)

M newspipe/web/controllers/feed.py => newspipe/web/controllers/feed.py +27 -20
@@ 16,22 16,26 @@ DEFAULT_MAX_ERROR = conf.DEFAULT_MAX_ERROR
class FeedController(AbstractController):
    _db_cls = Feed

    def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR,
                  limit=DEFAULT_LIMIT):
        return [feed for feed in self.read(
                            error_count__lt=max_error, enabled=True,
                            last_retrieved__lt=max_last)
                                .join(User).filter(User.is_active == True)
                                .order_by('last_retrieved')
                                .limit(limit)]
    def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT):
        return [
            feed
            for feed in self.read(
                error_count__lt=max_error, enabled=True, last_retrieved__lt=max_last
            )
            .join(User)
            .filter(User.is_active == True)
            .order_by("last_retrieved")
            .limit(limit)
        ]

    def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT):
        now = datetime.now()
        max_last = now - timedelta(minutes=60)
        feeds = self.list_late(max_last, max_error, limit)
        if feeds:
            self.update({'id__in': [feed.id for feed in feeds]},
                        {'last_retrieved': now})
            self.update(
                {"id__in": [feed.id for feed in feeds]}, {"last_retrieved": now}
            )
        return feeds

    def get_duplicates(self, feed_id):


@@ 43,8 47,9 @@ class FeedController(AbstractController):
        duplicates = []
        for pair in itertools.combinations(feed.articles[:1000], 2):
            date1, date2 = pair[0].date, pair[1].date
            if clear_string(pair[0].title) == clear_string(pair[1].title) \
                    and (date1 - date2) < timedelta(days=1):
            if clear_string(pair[0].title) == clear_string(pair[1].title) and (
                date1 - date2
            ) < timedelta(days=1):
                if pair[0].retrieved_date < pair[1].retrieved_date:
                    duplicates.append((pair[0], pair[1]))
                else:


@@ 75,11 80,11 @@ class FeedController(AbstractController):
        return self._count_by(Feed.link, filters)

    def _ensure_icon(self, attrs):
        if not attrs.get('icon_url'):
        if not attrs.get("icon_url"):
            return
        icon_contr = IconController()
        if not icon_contr.read(url=attrs['icon_url']).count():
            icon_contr.create(**{'url': attrs['icon_url']})
        if not icon_contr.read(url=attrs["icon_url"]).count():
            icon_contr.create(**{"url": attrs["icon_url"]})

    def create(self, **attrs):
        self._ensure_icon(attrs)


@@ 87,12 92,14 @@ class FeedController(AbstractController):

    def update(self, filters, attrs):
        from .article import ArticleController

        self._ensure_icon(attrs)
        if 'category_id' in attrs and attrs['category_id'] == 0:
            del attrs['category_id']
        elif 'category_id' in attrs:
        if "category_id" in attrs and attrs["category_id"] == 0:
            del attrs["category_id"]
        elif "category_id" in attrs:
            art_contr = ArticleController(self.user_id)
            for feed in self.read(**filters):
                art_contr.update({'feed_id': feed.id},
                                 {'category_id': attrs['category_id']})
                art_contr.update(
                    {"feed_id": feed.id}, {"category_id": attrs["category_id"]}
                )
        return super().update(filters, attrs)

M newspipe/web/controllers/icon.py => newspipe/web/controllers/icon.py +9 -5
@@ 9,11 9,15 @@ class IconController(AbstractController):
    _user_id_key = None

    def _build_from_url(self, attrs):
        if 'url' in attrs and 'content' not in attrs:
            resp = requests.get(attrs['url'], verify=False)
            attrs.update({'url': resp.url,
                    'mimetype': resp.headers.get('content-type', None),
                    'content': base64.b64encode(resp.content).decode('utf8')})
        if "url" in attrs and "content" not in attrs:
            resp = requests.get(attrs["url"], verify=False)
            attrs.update(
                {
                    "url": resp.url,
                    "mimetype": resp.headers.get("content-type", None),
                    "content": base64.b64encode(resp.content).decode("utf8"),
                }
            )
        return attrs

    def create(self, **attrs):

M newspipe/web/controllers/user.py => newspipe/web/controllers/user.py +5 -5
@@ 8,13 8,13 @@ logger = logging.getLogger(__name__)

class UserController(AbstractController):
    _db_cls = User
    _user_id_key = 'id'
    _user_id_key = "id"

    def _handle_password(self, attrs):
        if attrs.get('password'):
            attrs['pwdhash'] = generate_password_hash(attrs.pop('password'))
        elif 'password' in attrs:
            del attrs['password']
        if attrs.get("password"):
            attrs["pwdhash"] = generate_password_hash(attrs.pop("password"))
        elif "password" in attrs:
            del attrs["password"]

    def check_password(self, user, password):
        return check_password_hash(user.pwdhash, password)

M newspipe/web/decorators.py => newspipe/web/decorators.py +3 -0
@@ 13,9 13,11 @@ def async_maker(f):
    indexing the database) in background.
    This prevent the server to freeze.
    """

    def wrapper(*args, **kwargs):
        thr = Thread(target=f, args=args, kwargs=kwargs)
        thr.start()

    return wrapper




@@ 24,4 26,5 @@ def pyagg_default_decorator(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        return func(*args, **kwargs)

    return wrapper

M newspipe/web/forms.py => newspipe/web/forms.py +112 -62
@@ 30,8 30,17 @@ from flask import flash, url_for, redirect
from flask_wtf import FlaskForm
from flask_babel import lazy_gettext
from werkzeug.exceptions import NotFound
from wtforms import TextField, TextAreaField, PasswordField, BooleanField, \
        SubmitField, IntegerField, SelectField, validators, HiddenField
from wtforms import (
    TextField,
    TextAreaField,
    PasswordField,
    BooleanField,
    SubmitField,
    IntegerField,
    SelectField,
    validators,
    HiddenField,
)
from wtforms.fields.html5 import EmailField, URLField

from lib import misc_utils


@@ 43,27 52,44 @@ class SignupForm(FlaskForm):
    """
    Sign up form (registration to newspipe).
    """
    nickname = TextField(lazy_gettext("Nickname"),
            [validators.Required(lazy_gettext("Please enter your nickname."))])
    email = EmailField(lazy_gettext("Email"),
            [validators.Length(min=6, max=35),
             validators.Required(
                 lazy_gettext("Please enter your email address (only for account activation, won't be stored)."))])
    password = PasswordField(lazy_gettext("Password"),
            [validators.Required(lazy_gettext("Please enter a password.")),
             validators.Length(min=6, max=100)])

    nickname = TextField(
        lazy_gettext("Nickname"),
        [validators.Required(lazy_gettext("Please enter your nickname."))],
    )
    email = EmailField(
        lazy_gettext("Email"),
        [
            validators.Length(min=6, max=35),
            validators.Required(
                lazy_gettext(
                    "Please enter your email address (only for account activation, won't be stored)."
                )
            ),
        ],
    )
    password = PasswordField(
        lazy_gettext("Password"),
        [
            validators.Required(lazy_gettext("Please enter a password.")),
            validators.Length(min=6, max=100),
        ],
    )
    submit = SubmitField(lazy_gettext("Sign up"))

    def validate(self):
        ucontr = UserController()
        validated = super().validate()
        if ucontr.read(nickname=self.nickname.data).count():
            self.nickname.errors.append('Nickname already taken')
            self.nickname.errors.append("Nickname already taken")
            validated = False
        if self.nickname.data != User.make_valid_nickname(self.nickname.data):
            self.nickname.errors.append(lazy_gettext(
                    'This nickname has invalid characters. '
                    'Please use letters, numbers, dots and underscores only.'))
            self.nickname.errors.append(
                lazy_gettext(
                    "This nickname has invalid characters. "
                    "Please use letters, numbers, dots and underscores only."
                )
            )
            validated = False
        return validated



@@ 72,14 98,15 @@ class RedirectForm(FlaskForm):
    """
    Secure back redirects with WTForms.
    """

    next = HiddenField()

    def __init__(self, *args, **kwargs):
        FlaskForm.__init__(self, *args, **kwargs)
        if not self.next.data:
            self.next.data = misc_utils.get_redirect_target() or ''
            self.next.data = misc_utils.get_redirect_target() or ""

    def redirect(self, endpoint='home', **values):
    def redirect(self, endpoint="home", **values):
        if misc_utils.is_safe_url(self.next.data):
            return redirect(self.next.data)
        target = misc_utils.get_redirect_target()


@@ 90,13 117,21 @@ class SigninForm(RedirectForm):
    """
    Sign in form (connection to newspipe).
    """
    nickmane = TextField("Nickname",
                [validators.Length(min=3, max=35),
                validators.Required(
                lazy_gettext("Please enter your nickname."))])
    password = PasswordField(lazy_gettext('Password'),
            [validators.Required(lazy_gettext("Please enter a password.")),
             validators.Length(min=6, max=100)])

    nickmane = TextField(
        "Nickname",
        [
            validators.Length(min=3, max=35),
            validators.Required(lazy_gettext("Please enter your nickname.")),
        ],
    )
    password = PasswordField(
        lazy_gettext("Password"),
        [
            validators.Required(lazy_gettext("Please enter a password.")),
            validators.Length(min=6, max=100),
        ],
    )
    submit = SubmitField(lazy_gettext("Log In"))

    def __init__(self, *args, **kwargs):


@@ 109,15 144,14 @@ class SigninForm(RedirectForm):
        try:
            user = ucontr.get(nickname=self.nickmane.data)
        except NotFound:
            self.nickmane.errors.append(
                'Wrong nickname')
            self.nickmane.errors.append("Wrong nickname")
            validated = False
        else:
            if not user.is_active:
                self.nickmane.errors.append('Account not active')
                self.nickmane.errors.append("Account not active")
                validated = False
            if not ucontr.check_password(user, self.password.data):
                self.password.errors.append('Wrong password')
                self.password.errors.append("Wrong password")
                validated = False
            self.user = user
        return validated


@@ 127,19 161,24 @@ class UserForm(FlaskForm):
    """
    Create or edit a user (for the administrator).
    """
    nickname = TextField(lazy_gettext("Nickname"),
            [validators.Required(lazy_gettext("Please enter your nickname."))])

    nickname = TextField(
        lazy_gettext("Nickname"),
        [validators.Required(lazy_gettext("Please enter your nickname."))],
    )
    password = PasswordField(lazy_gettext("Password"))
    automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"),
                                default=True)
    automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"), default=True)
    submit = SubmitField(lazy_gettext("Save"))

    def validate(self):
        validated = super(UserForm, self).validate()
        if self.nickname.data != User.make_valid_nickname(self.nickname.data):
            self.nickname.errors.append(lazy_gettext(
                    'This nickname has invalid characters. '
                    'Please use letters, numbers, dots and underscores only.'))
            self.nickname.errors.append(
                lazy_gettext(
                    "This nickname has invalid characters. "
                    "Please use letters, numbers, dots and underscores only."
                )
            )
            validated = False
        return validated



@@ 148,17 187,18 @@ class ProfileForm(FlaskForm):
    """
    Edit user information.
    """
    nickname = TextField(lazy_gettext("Nickname"),
            [validators.Required(lazy_gettext("Please enter your nickname."))])

    nickname = TextField(
        lazy_gettext("Nickname"),
        [validators.Required(lazy_gettext("Please enter your nickname."))],
    )
    password = PasswordField(lazy_gettext("Password"))
    password_conf = PasswordField(lazy_gettext("Password Confirmation"))
    automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"),
                                default=True)
    automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"), default=True)
    bio = TextAreaField(lazy_gettext("Bio"))
    webpage = URLField(lazy_gettext("Webpage"))
    twitter = URLField(lazy_gettext("Twitter"))
    is_public_profile = BooleanField(lazy_gettext("Public profile"),
                                default=True)
    is_public_profile = BooleanField(lazy_gettext("Public profile"), default=True)
    submit = SubmitField(lazy_gettext("Save"))

    def validate(self):


@@ 169,28 209,34 @@ class ProfileForm(FlaskForm):
            self.password_conf.errors.append(message)
            validated = False
        if self.nickname.data != User.make_valid_nickname(self.nickname.data):
            self.nickname.errors.append(lazy_gettext('This nickname has '
                    'invalid characters. Please use letters, numbers, dots and'
                    ' underscores only.'))
            self.nickname.errors.append(
                lazy_gettext(
                    "This nickname has "
                    "invalid characters. Please use letters, numbers, dots and"
                    " underscores only."
                )
            )
            validated = False
        return validated


class AddFeedForm(FlaskForm):
    title = TextField(lazy_gettext("Title"), [validators.Optional()])
    link = TextField(lazy_gettext("Feed link"),
            [validators.Required(lazy_gettext("Please enter the URL."))])
    link = TextField(
        lazy_gettext("Feed link"),
        [validators.Required(lazy_gettext("Please enter the URL."))],
    )
    site_link = TextField(lazy_gettext("Site link"), [validators.Optional()])
    enabled = BooleanField(lazy_gettext("Check for updates"), default=True)
    submit = SubmitField(lazy_gettext("Save"))
    category_id = SelectField(lazy_gettext("Category of the feed"),
                              [validators.Optional()])
    category_id = SelectField(
        lazy_gettext("Category of the feed"), [validators.Optional()]
    )
    private = BooleanField(lazy_gettext("Private"), default=False)

    def set_category_choices(self, categories):
        self.category_id.choices = [('0', 'No Category')]
        self.category_id.choices += [(str(cat.id), cat.name)
                                      for cat in categories]
        self.category_id.choices = [("0", "No Category")]
        self.category_id.choices += [(str(cat.id), cat.name) for cat in categories]


class CategoryForm(FlaskForm):


@@ 199,13 245,13 @@ class CategoryForm(FlaskForm):


class BookmarkForm(FlaskForm):
    href = TextField(lazy_gettext("URL"),
                            [validators.Required(
                                        lazy_gettext("Please enter an URL."))])
    title = TextField(lazy_gettext("Title"),
                            [validators.Length(min=0, max=100)])
    description = TextField(lazy_gettext("Description"),
                            [validators.Length(min=0, max=500)])
    href = TextField(
        lazy_gettext("URL"), [validators.Required(lazy_gettext("Please enter an URL."))]
    )
    title = TextField(lazy_gettext("Title"), [validators.Length(min=0, max=100)])
    description = TextField(
        lazy_gettext("Description"), [validators.Length(min=0, max=500)]
    )
    tags = TextField(lazy_gettext("Tags"))
    to_read = BooleanField(lazy_gettext("To read"), default=False)
    shared = BooleanField(lazy_gettext("Shared"), default=False)


@@ 213,8 259,12 @@ class BookmarkForm(FlaskForm):


class InformationMessageForm(FlaskForm):
    subject = TextField(lazy_gettext("Subject"),
            [validators.Required(lazy_gettext("Please enter a subject."))])
    message = TextAreaField(lazy_gettext("Message"),
            [validators.Required(lazy_gettext("Please enter a content."))])
    subject = TextField(
        lazy_gettext("Subject"),
        [validators.Required(lazy_gettext("Please enter a subject."))],
    )
    message = TextAreaField(
        lazy_gettext("Message"),
        [validators.Required(lazy_gettext("Please enter a content."))],
    )
    submit = SubmitField(lazy_gettext("Send"))

M newspipe/web/lib/user_utils.py => newspipe/web/lib/user_utils.py +5 -7
@@ 1,22 1,20 @@


from itsdangerous import URLSafeTimedSerializer
import conf
from bootstrap import application


def generate_confirmation_token(nickname):
    serializer = URLSafeTimedSerializer(application.config['SECRET_KEY'])
    return serializer.dumps(nickname, salt=application.config['SECURITY_PASSWORD_SALT'])
    serializer = URLSafeTimedSerializer(application.config["SECRET_KEY"])
    return serializer.dumps(nickname, salt=application.config["SECURITY_PASSWORD_SALT"])


def confirm_token(token):
    serializer = URLSafeTimedSerializer(application.config['SECRET_KEY'])
    serializer = URLSafeTimedSerializer(application.config["SECRET_KEY"])
    try:
        nickname = serializer.loads(
            token,
            salt=application.config['SECURITY_PASSWORD_SALT'],
            max_age=conf.TOKEN_VALIDITY_PERIOD
            salt=application.config["SECURITY_PASSWORD_SALT"],
            max_age=conf.TOKEN_VALIDITY_PERIOD,
        )
    except:
        return False

M newspipe/web/lib/view_utils.py => newspipe/web/lib/view_utils.py +6 -4
@@ 15,12 15,14 @@ def etag_match(func):
            headers = {}
        else:
            return response
        if request.headers.get('if-none-match') == etag:
        if request.headers.get("if-none-match") == etag:
            response = Response(status=304)
            response.headers['Cache-Control'] \
                    = headers.get('Cache-Control', 'pragma: no-cache')
            response.headers["Cache-Control"] = headers.get(
                "Cache-Control", "pragma: no-cache"
            )
        elif not isinstance(response, Response):
            response = make_response(response)
        response.headers['etag'] = etag
        response.headers["etag"] = etag
        return response

    return wrapper

M newspipe/web/models/__init__.py => newspipe/web/models/__init__.py +20 -9
@@ 36,18 36,29 @@ from .tag import BookmarkTag
from .tag import ArticleTag
from .bookmark import Bookmark

__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon', 'Category',
            'Bookmark', 'ArticleTag', 'BookmarkTag']
__all__ = [
    "Feed",
    "Role",
    "User",
    "Article",
    "Icon",
    "Category",
    "Bookmark",
    "ArticleTag",
    "BookmarkTag",
]

import os

from sqlalchemy.engine import reflection
from sqlalchemy.schema import (
        MetaData,
        Table,
        DropTable,
        ForeignKeyConstraint,
        DropConstraint)
    MetaData,
    Table,
    DropTable,
    ForeignKeyConstraint,
    DropConstraint,
)


def db_empty(db):
    "Will drop every datas stocked in db."


@@ 71,9 82,9 @@ def db_empty(db):
    for table_name in inspector.get_table_names():
        fks = []
        for fk in inspector.get_foreign_keys(table_name):
            if not fk['name']:
            if not fk["name"]:
                continue
            fks.append(ForeignKeyConstraint((), (), name=fk['name']))
            fks.append(ForeignKeyConstraint((), (), name=fk["name"]))
        t = Table(table_name, metadata, *fks)
        tbs.append(t)
        all_fks.extend(fks)

M newspipe/web/models/article.py => newspipe/web/models/article.py +31 -17
@@ 48,40 48,54 @@ class Article(db.Model, RightMixin):
    retrieved_date = db.Column(db.DateTime(), default=datetime.utcnow)

    # foreign keys
    user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
    feed_id = db.Column(db.Integer(), db.ForeignKey('feed.id'))
    category_id = db.Column(db.Integer(), db.ForeignKey('category.id'))
    user_id = db.Column(db.Integer(), db.ForeignKey("user.id"))
    feed_id = db.Column(db.Integer(), db.ForeignKey("feed.id"))
    category_id = db.Column(db.Integer(), db.ForeignKey("category.id"))

    # relationships
    tag_objs = db.relationship('ArticleTag', back_populates='article',
                                 cascade='all,delete-orphan',
                                 lazy=False,
                                 foreign_keys='[ArticleTag.article_id]')
    tags = association_proxy('tag_objs', 'text')
    tag_objs = db.relationship(
        "ArticleTag",
        back_populates="article",
        cascade="all,delete-orphan",
        lazy=False,
        foreign_keys="[ArticleTag.article_id]",
    )
    tags = association_proxy("tag_objs", "text")

    # indexes
    #__table_args__ = (
    # __table_args__ = (
    #    Index('user_id'),
    #    Index('user_id', 'category_id'),
    #    Index('user_id', 'feed_id'),
    #    Index('ix_article_uid_fid_eid', user_id, feed_id, entry_id)
    #)
    # )

    # api whitelists
    @staticmethod
    def _fields_base_write():
        return {'readed', 'like', 'feed_id', 'category_id'}
        return {"readed", "like", "feed_id", "category_id"}

    @staticmethod
    def _fields_base_read():
        return {'id', 'entry_id', 'link', 'title', 'content', 'date',
                'retrieved_date', 'user_id', 'tags'}
        return {
            "id",
            "entry_id",
            "link",
            "title",
            "content",
            "date",
            "retrieved_date",
            "user_id",
            "tags",
        }

    @staticmethod
    def _fields_api_write():
        return {'tags'}
        return {"tags"}

    def __repr__(self):
        return "<Article(id=%d, entry_id=%s, title=%r, " \
               "date=%r, retrieved_date=%r)>" % (self.id, self.entry_id,
                       self.title, self.date, self.retrieved_date)
        return (
            "<Article(id=%d, entry_id=%s, title=%r, "
            "date=%r, retrieved_date=%r)>"
            % (self.id, self.entry_id, self.title, self.date, self.retrieved_date)
        )

M newspipe/web/models/bookmark.py => newspipe/web/models/bookmark.py +13 -9
@@ 40,6 40,7 @@ class Bookmark(db.Model, RightMixin):
    """
    Represent a bookmark.
    """

    id = db.Column(db.Integer(), primary_key=True)
    href = db.Column(db.String(), default="")
    title = db.Column(db.String(), default="")


@@ 47,22 48,25 @@ class Bookmark(db.Model, RightMixin):
    shared = db.Column(db.Boolean(), default=False)
    to_read = db.Column(db.Boolean(), default=False)
    time = db.Column(db.DateTime(), default=datetime.utcnow)
    user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
    user_id = db.Column(db.Integer(), db.ForeignKey("user.id"))

    # relationships
    tags = db.relationship(BookmarkTag, backref='of_bookmark', lazy='dynamic',
                               cascade='all,delete-orphan',
                               order_by=desc(BookmarkTag.text))
    tags_proxy = association_proxy('tags', 'text')

    tags = db.relationship(
        BookmarkTag,
        backref="of_bookmark",
        lazy="dynamic",
        cascade="all,delete-orphan",
        order_by=desc(BookmarkTag.text),
    )
    tags_proxy = association_proxy("tags", "text")

    @validates('description')
    @validates("description")
    def validates_title(self, key, value):
        return str(value).strip()

    @validates('extended')
    @validates("extended")
    def validates_description(self, key, value):
        return str(value).strip()

    def __repr__(self):
        return '<Bookmark %r>' % (self.href)
        return "<Bookmark %r>" % (self.href)

M newspipe/web/models/category.py => newspipe/web/models/category.py +6 -7
@@ 11,19 11,18 @@ class Category(db.Model, RightMixin):
    name = db.Column(db.String())

    # relationships
    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    feeds = db.relationship('Feed', cascade='all,delete-orphan')
    articles = db.relationship('Article',
                            cascade='all,delete-orphan')
    user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
    feeds = db.relationship("Feed", cascade="all,delete-orphan")
    articles = db.relationship("Article", cascade="all,delete-orphan")

    # index
    idx_category_uid = Index('user_id')
    idx_category_uid = Index("user_id")

    # api whitelists
    @staticmethod
    def _fields_base_read():
        return {'id', 'user_id'}
        return {"id", "user_id"}

    @staticmethod
    def _fields_base_write():
        return {'name'}
        return {"name"}

M newspipe/web/models/feed.py => newspipe/web/models/feed.py +29 -15
@@ 38,6 38,7 @@ class Feed(db.Model, RightMixin):
    """
    Represent a feed.
    """

    id = db.Column(db.Integer(), primary_key=True)
    title = db.Column(db.String(), default="")
    description = db.Column(db.String(), default="FR")


@@ 58,34 59,47 @@ class Feed(db.Model, RightMixin):
    error_count = db.Column(db.Integer(), default=0)

    # relationship
    icon_url = db.Column(db.String(), db.ForeignKey('icon.url'), default=None)
    user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
    category_id = db.Column(db.Integer(), db.ForeignKey('category.id'))
    articles = db.relationship(Article, backref='source', lazy='dynamic',
                               cascade='all,delete-orphan',
                               order_by=desc(Article.date))
    icon_url = db.Column(db.String(), db.ForeignKey("icon.url"), default=None)
    user_id = db.Column(db.Integer(), db.ForeignKey("user.id"))
    category_id = db.Column(db.Integer(), db.ForeignKey("category.id"))
    articles = db.relationship(
        Article,
        backref="source",
        lazy="dynamic",
        cascade="all,delete-orphan",
        order_by=desc(Article.date),
    )

    # index
    idx_feed_uid_cid = Index('user_id', 'category_id')
    idx_feed_uid = Index('user_id')
    idx_feed_uid_cid = Index("user_id", "category_id")
    idx_feed_uid = Index("user_id")

     # api whitelists
    # api whitelists
    @staticmethod
    def _fields_base_write():
        return {'title', 'description', 'link', 'site_link', 'enabled',
                'filters', 'last_error', 'error_count', 'category_id'}
        return {
            "title",
            "description",
            "link",
            "site_link",
            "enabled",
            "filters",
            "last_error",
            "error_count",
            "category_id",
        }

    @staticmethod
    def _fields_base_read():
        return {'id', 'user_id', 'icon_url', 'last_retrieved'}
        return {"id", "user_id", "icon_url", "last_retrieved"}

    @validates('title')
    @validates("title")
    def validates_title(self, key, value):
        return str(value).strip()

    @validates('description')
    @validates("description")
    def validates_description(self, key, value):
        return str(value).strip()

    def __repr__(self):
        return '<Feed %r>' % (self.title)
        return "<Feed %r>" % (self.title)

M newspipe/web/models/right_mixin.py => newspipe/web/models/right_mixin.py +14 -13
@@ 2,14 2,13 @@ from sqlalchemy.ext.associationproxy import _AssociationList


class RightMixin:

    @staticmethod
    def _fields_base_write():
        return set()

    @staticmethod
    def _fields_base_read():
        return set(['id'])
        return set(["id"])

    @staticmethod
    def _fields_api_write():


@@ 17,7 16,7 @@ class RightMixin:

    @staticmethod
    def _fields_api_read():
        return set(['id'])
        return set(["id"])

    @classmethod
    def fields_base_write(cls):


@@ 36,26 35,28 @@ class RightMixin:
        return cls.fields_base_read().union(cls._fields_api_read())

    def __getitem__(self, key):
        if not hasattr(self, '__dump__'):
        if not hasattr(self, "__dump__"):
            self.__dump__ = {}
        return self.__dump__.get(key)

    def __setitem__(self, key, value):
        if not hasattr(self, '__dump__'):
        if not hasattr(self, "__dump__"):
            self.__dump__ = {}
        self.__dump__[key] = value

    def dump(self, role='admin'):
        if role == 'admin':
            dico = {k: getattr(self, k)
                    for k in set(self.__table__.columns.keys())
                        .union(self.fields_api_read())
                        .union(self.fields_base_read())}
        elif role == 'api':
    def dump(self, role="admin"):
        if role == "admin":
            dico = {
                k: getattr(self, k)
                for k in set(self.__table__.columns.keys())
                .union(self.fields_api_read())
                .union(self.fields_base_read())
            }
        elif role == "api":
            dico = {k: getattr(self, k) for k in self.fields_api_read()}
        else:
            dico = {k: getattr(self, k) for k in self.fields_base_read()}
        if hasattr(self, '__dump__'):
        if hasattr(self, "__dump__"):
            dico.update(self.__dump__)
        for key, value in dico.items():  # preventing association proxy to die
            if isinstance(value, _AssociationList):

M newspipe/web/models/role.py => newspipe/web/models/role.py +2 -1
@@ 33,7 33,8 @@ class Role(db.Model):
    """
    Represent a role.
    """

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(), unique=True)

    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    user_id = db.Column(db.Integer, db.ForeignKey("user.id"))

M newspipe/web/models/tag.py => newspipe/web/models/tag.py +16 -8
@@ 8,12 8,14 @@ class ArticleTag(db.Model):
    text = db.Column(db.String, primary_key=True, unique=False)

    # foreign keys
    article_id = db.Column(db.Integer, db.ForeignKey('article.id', ondelete='CASCADE'),
                        primary_key=True)
    article_id = db.Column(
        db.Integer, db.ForeignKey("article.id", ondelete="CASCADE"), primary_key=True
    )

    # relationships
    article = db.relationship('Article', back_populates='tag_objs',
                                                    foreign_keys=[article_id])
    article = db.relationship(
        "Article", back_populates="tag_objs", foreign_keys=[article_id]
    )

    def __init__(self, text):
        self.text = text


@@ 24,12 26,18 @@ class BookmarkTag(db.Model):
    text = db.Column(db.String, unique=False)

    # foreign keys
    user_id = db.Column(db.Integer, db.ForeignKey('user.id', ondelete='CASCADE'))
    bookmark_id = db.Column(db.Integer, db.ForeignKey('bookmark.id', ondelete='CASCADE'))
    user_id = db.Column(db.Integer, db.ForeignKey("user.id", ondelete="CASCADE"))
    bookmark_id = db.Column(
        db.Integer, db.ForeignKey("bookmark.id", ondelete="CASCADE")
    )

    # relationships
    bookmark = db.relationship('Bookmark', back_populates='tags',
                            cascade="all,delete", foreign_keys=[bookmark_id])
    bookmark = db.relationship(
        "Bookmark",
        back_populates="tags",
        cascade="all,delete",
        foreign_keys=[bookmark_id],
    )

    # def __init__(self, text, user_id):
    #     self.text = text

M newspipe/web/models/user.py => newspipe/web/models/user.py +19 -13
@@ 44,6 44,7 @@ class User(db.Model, UserMixin, RightMixin):
    """
    Represent a user.
    """

    id = db.Column(db.Integer, primary_key=True)
    nickname = db.Column(db.String(), unique=True)
    pwdhash = db.Column(db.String())


@@ 64,29 65,34 @@ class User(db.Model, UserMixin, RightMixin):
    is_api = db.Column(db.Boolean(), default=False)

    # relationships
    categories = db.relationship('Category', backref='user',
                              cascade='all, delete-orphan',
                            foreign_keys=[Category.user_id])
    feeds = db.relationship('Feed', backref='user',
                         cascade='all, delete-orphan',
                            foreign_keys=[Feed.user_id])
    categories = db.relationship(
        "Category",
        backref="user",
        cascade="all, delete-orphan",
        foreign_keys=[Category.user_id],
    )
    feeds = db.relationship(
        "Feed",
        backref="user",
        cascade="all, delete-orphan",
        foreign_keys=[Feed.user_id],
    )

    @staticmethod
    def _fields_base_write():
        return {'login', 'password'}
        return {"login", "password"}

    @staticmethod
    def _fields_base_read():
        return {'date_created', 'last_connection'}
        return {"date_created", "last_connection"}

    @staticmethod
    def make_valid_nickname(nickname):
        return re.sub('[^a-zA-Z0-9_\.]', '', nickname)
        return re.sub("[^a-zA-Z0-9_\.]", "", nickname)

    @validates('bio')
    @validates("bio")
    def validates_bio(self, key, value):
        assert len(value) <= 5000, \
                AssertionError("maximum length for bio: 5000")
        assert len(value) <= 5000, AssertionError("maximum length for bio: 5000")
        return value.strip()

    def get_id(self):


@@ 105,4 111,4 @@ class User(db.Model, UserMixin, RightMixin):
        return self.id == other.id

    def __repr__(self):
        return '<User %r>' % (self.nickname)
        return "<User %r>" % (self.nickname)

M newspipe/web/views/__init__.py => newspipe/web/views/__init__.py +19 -4
@@ 8,10 8,25 @@ from web.views.admin import admin_bp
from web.views.user import user_bp, users_bp
from web.views.bookmark import bookmark_bp, bookmarks_bp

__all__ = ['views', 'home', 'session_mgmt', 'v2', 'v3',
           'article_bp', 'articles_bp', 'feed_bp', 'feeds_bp',
           'category_bp', 'categories_bp', 'icon_bp',
           'admin_bp', 'user_bp', 'users_bp', 'bookmark_bp', 'bookmarks_bp']
__all__ = [
    "views",
    "home",
    "session_mgmt",
    "v2",
    "v3",
    "article_bp",
    "articles_bp",
    "feed_bp",
    "feeds_bp",
    "category_bp",
    "categories_bp",
    "icon_bp",
    "admin_bp",
    "user_bp",
    "users_bp",
    "bookmark_bp",
    "bookmarks_bp",
]

import conf
from flask import request

M newspipe/web/views/admin.py => newspipe/web/views/admin.py +71 -49
@@ 1,5 1,5 @@
from datetime import datetime
from flask import (Blueprint, render_template, redirect, flash, url_for)
from flask import Blueprint, render_template, redirect, flash, url_for
from flask_babel import gettext, format_timedelta
from flask_login import login_required, current_user



@@ 8,41 8,45 @@ from web.views.common import admin_permission
from web.controllers import UserController
from web.forms import InformationMessageForm, UserForm

admin_bp = Blueprint('admin', __name__, url_prefix='/admin')
admin_bp = Blueprint("admin", __name__, url_prefix="/admin")


@admin_bp.route('/dashboard', methods=['GET', 'POST'])
@admin_bp.route("/dashboard", methods=["GET", "POST"])
@login_required
@admin_permission.require(http_exception=403)
def dashboard():
    last_cons, now = {}, datetime.utcnow()
    users = list(UserController().read().order_by('id'))
    users = list(UserController().read().order_by("id"))
    form = InformationMessageForm()
    for user in users:
        last_cons[user.id] = format_timedelta(now - user.last_seen)
    return render_template('admin/dashboard.html', now=datetime.utcnow(),
            last_cons=last_cons, users=users, current_user=current_user,
            form=form)


@admin_bp.route('/user/create', methods=['GET'])
@admin_bp.route('/user/edit/<int:user_id>', methods=['GET'])
    return render_template(
        "admin/dashboard.html",
        now=datetime.utcnow(),
        last_cons=last_cons,
        users=users,
        current_user=current_user,
        form=form,
    )


@admin_bp.route("/user/create", methods=["GET"])
@admin_bp.route("/user/edit/<int:user_id>", methods=["GET"])
@login_required
@admin_permission.require(http_exception=403)
def user_form(user_id=None):
    if user_id is not None:
        user = UserController().get(id=user_id)
        form = UserForm(obj=user)
        message = gettext('Edit the user <i>%(nick)s</i>', nick=user.nickname)
        message = gettext("Edit the user <i>%(nick)s</i>", nick=user.nickname)
    else:
        form = UserForm()
        message = gettext('Add a new user')
    return render_template('/admin/create_user.html',
                           form=form, message=message)
        message = gettext("Add a new user")
    return render_template("/admin/create_user.html", form=form, message=message)


@admin_bp.route('/user/create', methods=['POST'])
@admin_bp.route('/user/edit/<int:user_id>', methods=['POST'])
@admin_bp.route("/user/create", methods=["POST"])
@admin_bp.route("/user/edit/<int:user_id>", methods=["POST"])
@login_required
@admin_permission.require(http_exception=403)
def process_user_form(user_id=None):


@@ 53,31 57,42 @@ def process_user_form(user_id=None):
    user_contr = UserController()

    if not form.validate():
        return render_template('/admin/create_user.html', form=form,
                               message=gettext('Some errors were found'))
        return render_template(
            "/admin/create_user.html",
            form=form,
            message=gettext("Some errors were found"),
        )

    if user_id is not None:
        # Edit a user
        user_contr.update({'id': user_id},
                          {'nickname': form.nickname.data,
                           'password': form.password.data,
                           'automatic_crawling': form.automatic_crawling.data})
        user_contr.update(
            {"id": user_id},
            {
                "nickname": form.nickname.data,
                "password": form.password.data,
                "automatic_crawling": form.automatic_crawling.data,
            },
        )
        user = user_contr.get(id=user_id)
        flash(gettext('User %(nick)s successfully updated',
                      nick=user.nickname), 'success')
        flash(
            gettext("User %(nick)s successfully updated", nick=user.nickname), "success"
        )
    else:
        # Create a new user (by the admin)
        user = user_contr.create(nickname=form.nickname.data,
                            password=form.password.data,
                            automatic_crawling=form.automatic_crawling.data,
                            is_admin=False,
                            is_active=True)
        flash(gettext('User %(nick)s successfully created',
                      nick=user.nickname), 'success')
    return redirect(url_for('admin.user_form', user_id=user.id))
        user = user_contr.create(
            nickname=form.nickname.data,
            password=form.password.data,
            automatic_crawling=form.automatic_crawling.data,
            is_admin=False,
            is_active=True,
        )
        flash(
            gettext("User %(nick)s successfully created", nick=user.nickname), "success"
        )
    return redirect(url_for("admin.user_form", user_id=user.id))


@admin_bp.route('/delete_user/<int:user_id>', methods=['GET'])
@admin_bp.route("/delete_user/<int:user_id>", methods=["GET"])
@login_required
@admin_permission.require(http_exception=403)
def delete_user(user_id=None):


@@ 86,16 101,21 @@ def delete_user(user_id=None):
    """
    try:
        user = UserController().delete(user_id)
        flash(gettext('User %(nick)s successfully deleted',
                      nick=user.nickname), 'success')
        flash(
            gettext("User %(nick)s successfully deleted", nick=user.nickname), "success"
        )
    except Exception as error:
        flash(
            gettext('An error occurred while trying to delete a user: %(error)s',
                        error=error), 'danger')
    return redirect(url_for('admin.dashboard'))
            gettext(
                "An error occurred while trying to delete a user: %(error)s",
                error=error,
            ),
            "danger",
        )
    return redirect(url_for("admin.dashboard"))


@admin_bp.route('/toggle_user/<int:user_id>', methods=['GET'])
@admin_bp.route("/toggle_user/<int:user_id>", methods=["GET"])
@login_required
@admin_permission.require()
def toggle_user(user_id=None):


@@ 104,16 124,18 @@ def toggle_user(user_id=None):
    """
    ucontr = UserController()
    user = ucontr.get(id=user_id)
    user_changed = ucontr.update({'id': user_id},
            {'is_active': not user.is_active})
    user_changed = ucontr.update({"id": user_id}, {"is_active": not user.is_active})

    if not user_changed:
        flash(gettext('This user does not exist.'), 'danger')
        return redirect(url_for('admin.dashboard'))
        flash(gettext("This user does not exist."), "danger")
        return redirect(url_for("admin.dashboard"))

    else:
        act_txt = 'activated' if user.is_active else 'desactivated'
        message = gettext('User %(nickname)s successfully %(is_active)s',
                          nickname=user.nickname, is_active=act_txt)
    flash(message, 'success')
    return redirect(url_for('admin.dashboard'))
        act_txt = "activated" if user.is_active else "desactivated"
        message = gettext(
            "User %(nickname)s successfully %(is_active)s",
            nickname=user.nickname,
            is_active=act_txt,
        )
    flash(message, "success")
    return redirect(url_for("admin.dashboard"))

M newspipe/web/views/api/v2/__init__.py => newspipe/web/views/api/v2/__init__.py +1 -1
@@ 1,3 1,3 @@
from web.views.api.v2 import article, feed, category

__all__ = ['article', 'feed', 'category']
__all__ = ["article", "feed", "category"]

M newspipe/web/views/api/v2/article.py => newspipe/web/views/api/v2/article.py +19 -13
@@ 6,8 6,12 @@ from flask_restful import Api

from web.views.common import api_permission
from web.controllers import ArticleController
from web.views.api.v2.common import (PyAggAbstractResource,
        PyAggResourceNew, PyAggResourceExisting, PyAggResourceMulti)
from web.views.api.v2.common import (
    PyAggAbstractResource,
    PyAggResourceNew,
    PyAggResourceExisting,
    PyAggResourceMulti,
)


class ArticleNewAPI(PyAggResourceNew):


@@ 24,30 28,32 @@ class ArticlesAPI(PyAggResourceMulti):

class ArticlesChallenge(PyAggAbstractResource):
    controller_cls = ArticleController
    attrs = {'ids': {'type': list, 'default': []}}
    attrs = {"ids": {"type": list, "default": []}}

    @api_permission.require(http_exception=403)
    def get(self):
        parsed_args = self.reqparse_args(right='read')
        parsed_args = self.reqparse_args(right="read")
        # collecting all attrs for casting purpose
        attrs = self.controller_cls._get_attrs_desc('admin')
        for id_dict in parsed_args['ids']:
        attrs = self.controller_cls._get_attrs_desc("admin")
        for id_dict in parsed_args["ids"]:
            keys_to_ignore = []
            for key in id_dict:
                if key not in attrs:
                    keys_to_ignore.append(key)
                if issubclass(attrs[key]['type'], datetime):
                if issubclass(attrs[key]["type"], datetime):
                    id_dict[key] = dateutil.parser.parse(id_dict[key])
            for key in keys_to_ignore:
                del id_dict[key]

        result = list(self.controller.challenge(parsed_args['ids']))
        result = list(self.controller.challenge(parsed_args["ids"]))
        return result or None, 200 if result else 204


api = Api(current_app, prefix=API_ROOT)

api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json')
api.add_resource(ArticleAPI, '/article/<int:obj_id>', endpoint='article.json')
api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json')
api.add_resource(ArticlesChallenge, '/articles/challenge',
                 endpoint='articles_challenge.json')
api.add_resource(ArticleNewAPI, "/article", endpoint="article_new.json")
api.add_resource(ArticleAPI, "/article/<int:obj_id>", endpoint="article.json")
api.add_resource(ArticlesAPI, "/articles", endpoint="articles.json")
api.add_resource(
    ArticlesChallenge, "/articles/challenge", endpoint="articles_challenge.json"
)

M newspipe/web/views/api/v2/category.py => newspipe/web/views/api/v2/category.py +8 -7
@@ 3,9 3,11 @@ from flask import current_app
from flask_restful import Api

from web.controllers.category import CategoryController
from web.views.api.v2.common import (PyAggResourceNew,
                                  PyAggResourceExisting,
                                  PyAggResourceMulti)
from web.views.api.v2.common import (
    PyAggResourceNew,
    PyAggResourceExisting,
    PyAggResourceMulti,
)


class CategoryNewAPI(PyAggResourceNew):


@@ 21,7 23,6 @@ class CategoriesAPI(PyAggResourceMulti):


api = Api(current_app, prefix=API_ROOT)
api.add_resource(CategoryNewAPI, '/category', endpoint='category_new.json')
api.add_resource(CategoryAPI, '/category/<int:obj_id>',
                 endpoint='category.json')
api.add_resource(CategoriesAPI, '/categories', endpoint='categories.json')
api.add_resource(CategoryNewAPI, "/category", endpoint="category_new.json")
api.add_resource(CategoryAPI, "/category/<int:obj_id>", endpoint="category.json")
api.add_resource(CategoriesAPI, "/categories", endpoint="categories.json")

M newspipe/web/views/api/v2/common.py => newspipe/web/views/api/v2/common.py +35 -29
@@ 26,8 26,12 @@ from flask import request
from flask_restful import Resource, reqparse
from flask_login import current_user

from web.views.common import admin_permission, api_permission, \
                             login_user_bundle, jsonify
from web.views.common import (
    admin_permission,
    api_permission,
    login_user_bundle,
    jsonify,
)
from web.controllers import UserController

logger = logging.getLogger(__name__)


@@ 50,6 54,7 @@ def authenticate(func):
        if current_user.is_authenticated:
            return func(*args, **kwargs)
        raise Unauthorized()

    return wrapper




@@ 64,8 69,9 @@ class PyAggAbstractResource(Resource):
            return self.controller_cls()
        return self.controller_cls(current_user.id)

    def reqparse_args(self, right, req=None, strict=False, default=True,
                      allow_empty=False):
    def reqparse_args(
        self, right, req=None, strict=False, default=True, allow_empty=False
    ):
        """
        strict: bool
            if True will throw 400 error if args are defined and not in request


@@ 89,42 95,41 @@ class PyAggAbstractResource(Resource):
        if self.attrs is not None:
            attrs = self.attrs
        elif admin_permission.can():
            attrs = self.controller_cls._get_attrs_desc('admin')
            attrs = self.controller_cls._get_attrs_desc("admin")
        elif api_permission.can():
            attrs = self.controller_cls._get_attrs_desc('api', right)
            attrs = self.controller_cls._get_attrs_desc("api", right)
        else:
            attrs = self.controller_cls._get_attrs_desc('base', right)
            attrs = self.controller_cls._get_attrs_desc("base", right)
        assert attrs, "No defined attrs for %s" % self.__class__.__name__

        for attr_name, attr in attrs.items():
            if not default and attr_name not in in_values:
                continue
            else:
                parser.add_argument(attr_name, location='json',
                                        default=in_values[attr_name])
                parser.add_argument(
                    attr_name, location="json", default=in_values[attr_name]
                )
        return parser.parse_args(req=request.args, strict=strict)


class PyAggResourceNew(PyAggAbstractResource):

    @api_permission.require(http_exception=403)
    def post(self):
        """Create a single new object"""
        return self.controller.create(**self.reqparse_args(right='write')), 201
        return self.controller.create(**self.reqparse_args(right="write")), 201


class PyAggResourceExisting(PyAggAbstractResource):

    def get(self, obj_id=None):
        """Retrieve a single object"""
        return self.controller.get(id=obj_id)

    def put(self, obj_id=None):
        """update an object, new attrs should be passed in the payload"""
        args = self.reqparse_args(right='write', default=False)
        args = self.reqparse_args(right="write", default=False)
        if not args:
            raise BadRequest()
        return self.controller.update({'id': obj_id}, args), 200
        return self.controller.update({"id": obj_id}, args), 200

    def delete(self, obj_id=None):
        """delete a object"""


@@ 133,19 138,18 @@ class PyAggResourceExisting(PyAggAbstractResource):


class PyAggResourceMulti(PyAggAbstractResource):

    def get(self):
        """retrieve several objects. filters can be set in the payload on the
        different fields of the object, and a limit can be set in there as well
        """
        args = {}
        try:
            limit = request.json.pop('limit', 10)
            order_by = request.json.pop('order_by', None)
            limit = request.json.pop("limit", 10)
            order_by = request.json.pop("order_by", None)
        except Exception:
            args = self.reqparse_args(right='read', default=False)
            limit = request.args.get('limit', 10)
            order_by = request.args.get('order_by', None)
            args = self.reqparse_args(right="read", default=False)
            limit = request.args.get("limit", 10)
            order_by = request.args.get("order_by", None)
        query = self.controller.read(**args)
        if order_by:
            query = query.order_by(order_by)


@@ 163,10 167,11 @@ class PyAggResourceMulti(PyAggAbstractResource):

        class Proxy:
            pass

        for attrs in request.json:
            try:
                Proxy.json = attrs
                args = self.reqparse_args('write', req=Proxy, default=False)
                args = self.reqparse_args("write", req=Proxy, default=False)
                obj = self.controller.create(**args)
                results.append(obj)
            except Exception as error:


@@ 188,20 193,21 @@ class PyAggResourceMulti(PyAggAbstractResource):

        class Proxy:
            pass

        for obj_id, attrs in request.json:
            try:
                Proxy.json = attrs
                args = self.reqparse_args('write', req=Proxy, default=False)
                result = self.controller.update({'id': obj_id}, args)
                args = self.reqparse_args("write", req=Proxy, default=False)
                result = self.controller.update({"id": obj_id}, args)
                if result:
                    results.append('ok')
                    results.append("ok")
                else:
                    results.append('nok')
                    results.append("nok")
            except Exception as error:
                results.append(str(error))
        if results.count('ok') == 0:  # all failed => 500
        if results.count("ok") == 0:  # all failed => 500
            status = 500
        elif results.count('ok') != len(results):  # some failed => 206
        elif results.count("ok") != len(results):  # some failed => 206
            status = 206
        return results, status



@@ 212,11 218,11 @@ class PyAggResourceMulti(PyAggAbstractResource):
        for obj_id in request.json:
            try:
                self.controller.delete(obj_id)
                results.append('ok')
                results.append("ok")
            except Exception as error:
                status = 206
                results.append(error)
        # if no operation succeeded, it's not partial anymore, returning err 500
        if status == 206 and results.count('ok') == 0:
        if status == 206 and results.count("ok") == 0:
            status = 500
        return results, status

M newspipe/web/views/api/v2/feed.py => newspipe/web/views/api/v2/feed.py +17 -17
@@ 3,14 3,14 @@ from flask import current_app
from flask_restful import Api

from web.views.common import api_permission
from web.controllers.feed import (FeedController,
                                  DEFAULT_MAX_ERROR,
                                  DEFAULT_LIMIT)
from web.controllers.feed import FeedController, DEFAULT_MAX_ERROR, DEFAULT_LIMIT

from web.views.api.v2.common import PyAggAbstractResource, \
                                 PyAggResourceNew, \
                                 PyAggResourceExisting, \
                                 PyAggResourceMulti
from web.views.api.v2.common import (
    PyAggAbstractResource,
    PyAggResourceNew,
    PyAggResourceExisting,
    PyAggResourceMulti,
)


class FeedNewAPI(PyAggResourceNew):


@@ 27,21 27,21 @@ class FeedsAPI(PyAggResourceMulti):

class FetchableFeedAPI(PyAggAbstractResource):
    controller_cls = FeedController
    attrs = {'max_error': {'type': int, 'default': DEFAULT_MAX_ERROR},
             'limit': {'type': int, 'default': DEFAULT_LIMIT}}
    attrs = {
        "max_error": {"type": int, "default": DEFAULT_MAX_ERROR},
        "limit": {"type": int, "default": DEFAULT_LIMIT},
    }

    @api_permission.require(http_exception=403)
    def get(self):
        args = self.reqparse_args(right='read', allow_empty=True)
        result = [feed for feed
                  in self.controller.list_fetchable(**args)]
        args = self.reqparse_args(right="read", allow_empty=True)
        result = [feed for feed in self.controller.list_fetchable(**args)]
        return result or None, 200 if result else 204


api = Api(current_app, prefix=API_ROOT)

api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json')
api.add_resource(FeedAPI, '/feed/<int:obj_id>', endpoint='feed.json')
api.add_resource(FeedsAPI, '/feeds', endpoint='feeds.json')
api.add_resource(FetchableFeedAPI, '/feeds/fetchable',
                 endpoint='fetchable_feed.json')
api.add_resource(FeedNewAPI, "/feed", endpoint="feed_new.json")
api.add_resource(FeedAPI, "/feed/<int:obj_id>", endpoint="feed.json")
api.add_resource(FeedsAPI, "/feeds", endpoint="feeds.json")
api.add_resource(FetchableFeedAPI, "/feeds/fetchable", endpoint="fetchable_feed.json")

M newspipe/web/views/api/v3/__init__.py => newspipe/web/views/api/v3/__init__.py +1 -1
@@ 1,3 1,3 @@
from web.views.api.v3 import article

__all__ = ['article']
__all__ = ["article"]

M newspipe/web/views/api/v3/article.py => newspipe/web/views/api/v3/article.py +18 -18
@@ 35,6 35,7 @@ from web.controllers import ArticleController, FeedController
from web.views.api.v3.common import AbstractProcessor
from web.views.api.v3.common import url_prefix, auth_func


class ArticleProcessor(AbstractProcessor):
    """Concrete processors for the Article Web service.
    """


@@ 43,7 44,7 @@ class ArticleProcessor(AbstractProcessor):
        try:
            article = ArticleController(current_user.id).get(id=instance_id)
        except NotFound:
            raise ProcessingException(description='No such article.', code=404)
            raise ProcessingException(description="No such article.", code=404)
        self.is_authorized(current_user, article)

    def post_preprocessor(self, data=None, **kw):


@@ 52,7 53,7 @@ class ArticleProcessor(AbstractProcessor):
        try:
            feed = FeedController(current_user.id).get(id=data["feed_id"])
        except NotFound:
            raise ProcessingException(description='No such feed.', code=404)
            raise ProcessingException(description="No such feed.", code=404)
        self.is_authorized(current_user, feed)

        data["category_id"] = feed.category_id


@@ 61,24 62,23 @@ class ArticleProcessor(AbstractProcessor):
        try:
            article = ArticleController(current_user.id).get(id=instance_id)
        except NotFound:
            raise ProcessingException(description='No such article.', code=404)
            raise ProcessingException(description="No such article.", code=404)
        self.is_authorized(current_user, article)


article_processor = ArticleProcessor()

blueprint_article = manager.create_api_blueprint(models.Article,
        url_prefix=url_prefix,
        methods=['GET', 'POST', 'PUT', 'DELETE'],
        preprocessors=dict(GET_SINGLE=[auth_func,
                                    article_processor.get_single_preprocessor],
                            GET_MANY=[auth_func,
                                    article_processor.get_many_preprocessor],
                            POST=[auth_func,
                                    article_processor.post_preprocessor],
                            PUT_SINGLE=[auth_func,
                                    article_processor.put_single_preprocessor],
                            PUT_MANY=[auth_func,
                                    article_processor.put_many_preprocessor],
                            DELETE=[auth_func,
                                    article_processor.delete_preprocessor]))
blueprint_article = manager.create_api_blueprint(
    models.Article,
    url_prefix=url_prefix,
    methods=["GET", "POST", "PUT", "DELETE"],
    preprocessors=dict(
        GET_SINGLE=[auth_func, article_processor.get_single_preprocessor],
        GET_MANY=[auth_func, article_processor.get_many_preprocessor],
        POST=[auth_func, article_processor.post_preprocessor],
        PUT_SINGLE=[auth_func, article_processor.put_single_preprocessor],
        PUT_MANY=[auth_func, article_processor.put_many_preprocessor],
        DELETE=[auth_func, article_processor.delete_preprocessor],
    ),
)
application.register_blueprint(blueprint_article)

M newspipe/web/views/api/v3/common.py => newspipe/web/views/api/v3/common.py +12 -16
@@ 33,7 33,8 @@ from werkzeug.exceptions import NotFound
from web.controllers import ArticleController, UserController
from web.views.common import login_user_bundle

url_prefix = '/api/v3'
url_prefix = "/api/v3"


def auth_func(*args, **kw):
    if request.authorization:


@@ 41,24 42,23 @@ def auth_func(*args, **kw):
        try:
            user = ucontr.get(nickname=request.authorization.username)
        except NotFound:
            raise ProcessingException("Couldn't authenticate your user",
                                        code=401)
            raise ProcessingException("Couldn't authenticate your user", code=401)
        if not ucontr.check_password(user, request.authorization.password):
            raise ProcessingException("Couldn't authenticate your user",
                                        code=401)
            raise ProcessingException("Couldn't authenticate your user", code=401)
        if not user.is_active:
            raise ProcessingException("User is deactivated", code=401)
        login_user_bundle(user)
    if not current_user.is_authenticated:
        raise ProcessingException(description='Not authenticated!', code=401)
        raise ProcessingException(description="Not authenticated!", code=401)


class AbstractProcessor():
class AbstractProcessor:
    """Abstract processors for the Web services.
    """

    def is_authorized(self, user, obj):
        if user.id != obj.user_id:
            raise ProcessingException(description='Not Authorized', code=401)
            raise ProcessingException(description="Not Authorized", code=401)

    def get_single_preprocessor(self, instance_id=None, **kw):
        # Check if the user is authorized to modify the specified


@@ 69,13 69,11 @@ class AbstractProcessor():
        """Accepts a single argument, `search_params`, which is a dictionary
        containing the search parameters for the request.
        """
        filt = dict(name="user_id",
                    op="eq",
                    val=current_user.id)
        filt = dict(name="user_id", op="eq", val=current_user.id)

        # Check if there are any filters there already.
        if "filters" not in search_params:
          search_params["filters"] = []
            search_params["filters"] = []

        search_params["filters"].append(filt)



@@ 95,13 93,11 @@ class AbstractProcessor():
        is a dictionary representing the fields to change on the matching
        instances and the values to which they will be set.
        """
        filt = dict(name="user_id",
                    op="eq",
                    val=current_user.id)
        filt = dict(name="user_id", op="eq", val=current_user.id)

        # Check if there are any filters there already.
        if "filters" not in search_params:
          search_params["filters"] = []
            search_params["filters"] = []

        search_params["filters"].append(filt)


M newspipe/web/views/api/v3/feed.py => newspipe/web/views/api/v3/feed.py +14 -10
@@ 33,6 33,7 @@ from web.controllers import FeedController
from web.views.api.v3.common import AbstractProcessor
from web.views.api.v3.common import url_prefix, auth_func


class FeedProcessor(AbstractProcessor):
    """Concrete processors for the Feed Web service.
    """


@@ 43,16 44,19 @@ class FeedProcessor(AbstractProcessor):
        feed = FeedController(current_user.id).get(id=instance_id)
        self.is_authorized(current_user, feed)


feed_processor = FeedProcessor()

blueprint_feed = manager.create_api_blueprint(models.Feed,
        url_prefix=url_prefix,
        methods=['GET', 'POST', 'PUT', 'DELETE'],
        preprocessors=dict(GET_SINGLE=[auth_func,
                                    feed_processor.get_single_preprocessor],
                           GET_MANY=[auth_func,
                                    feed_processor.get_many_preprocessor],
                           PUT_SINGLE=[auth_func],
                           POST=[auth_func],
                           DELETE=[auth_func]))
blueprint_feed = manager.create_api_blueprint(
    models.Feed,
    url_prefix=url_prefix,
    methods=["GET", "POST", "PUT", "DELETE"],
    preprocessors=dict(
        GET_SINGLE=[auth_func, feed_processor.get_single_preprocessor],
        GET_MANY=[auth_func, feed_processor.get_many_preprocessor],
        PUT_SINGLE=[auth_func],
        POST=[auth_func],
        DELETE=[auth_func],
    ),
)
application.register_blueprint(blueprint_feed)

M newspipe/web/views/article.py => newspipe/web/views/article.py +63 -52
@@ 1,6 1,14 @@
from datetime import datetime, timedelta
from flask import (Blueprint, g, render_template, redirect,
                   flash, url_for, make_response, request)
from flask import (
    Blueprint,
    g,
    render_template,
    redirect,
    flash,
    url_for,
    make_response,
    request,
)

from flask_babel import gettext
from flask_login import login_required, current_user


@@ 9,25 17,24 @@ from flask_login import login_required, current_user
from bootstrap import db
from lib.utils import clear_string, redirect_url
from lib.data import export_json
from web.controllers import (ArticleController, UserController,
                            CategoryController)
from web.controllers import ArticleController, UserController, CategoryController
from web.lib.view_utils import etag_match

articles_bp = Blueprint('articles', __name__, url_prefix='/articles')
article_bp = Blueprint('article', __name__, url_prefix='/article')
articles_bp = Blueprint("articles", __name__, url_prefix="/articles")
article_bp = Blueprint("article", __name__, url_prefix="/article")


@article_bp.route('/redirect/<int:article_id>', methods=['GET'])
@article_bp.route("/redirect/<int:article_id>", methods=["GET"])
@login_required
def redirect_to_article(article_id):
    contr = ArticleController(current_user.id)
    article = contr.get(id=article_id)
    if not article.readed:
        contr.update({'id': article.id}, {'readed': True})
        contr.update({"id": article.id}, {"readed": True})
    return redirect(article.link)


@article_bp.route('/<int:article_id>', methods=['GET'])
@article_bp.route("/<int:article_id>", methods=["GET"])
@login_required
@etag_match
def article(article_id=None):


@@ 35,11 42,12 @@ def article(article_id=None):
    Presents an article.
    """
    article = ArticleController(current_user.id).get(id=article_id)
    return render_template('article.html',
                           head_titles=[clear_string(article.title)],
                           article=article)
    return render_template(
        "article.html", head_titles=[clear_string(article.title)], article=article
    )

@article_bp.route('/public/<int:article_id>', methods=['GET'])

@article_bp.route("/public/<int:article_id>", methods=["GET"])
@etag_match
def article_pub(article_id=None):
    """


@@ 48,13 56,13 @@ def article_pub(article_id=None):
    """
    article = ArticleController().get(id=article_id)
    if article.source.private or not article.source.user.is_public_profile:
        return render_template('errors/404.html'), 404
    return render_template('article_pub.html',
                           head_titles=[clear_string(article.title)],
                           article=article)
        return render_template("errors/404.html"), 404
    return render_template(
        "article_pub.html", head_titles=[clear_string(article.title)], article=article
    )


@article_bp.route('/like/<int:article_id>', methods=['GET'])
@article_bp.route("/like/<int:article_id>", methods=["GET"])
@login_required
def like(article_id=None):
    """


@@ 62,80 70,84 @@ def like(article_id=None):
    """
    art_contr = ArticleController(current_user.id)
    article = art_contr.get(id=article_id)
    art_contr = art_contr.update({'id': article_id},
                                 {'like': not article.like})
    art_contr = art_contr.update({"id": article_id}, {"like": not article.like})
    return redirect(redirect_url())


@article_bp.route('/delete/<int:article_id>', methods=['GET'])
@article_bp.route("/delete/<int:article_id>", methods=["GET"])
@login_required
def delete(article_id=None):
    """
    Delete an article from the database.
    """
    article = ArticleController(current_user.id).delete(article_id)
    flash(gettext('Article %(article_title)s deleted',
                  article_title=article.title), 'success')
    return redirect(url_for('home'))
    flash(
        gettext("Article %(article_title)s deleted", article_title=article.title),
        "success",
    )
    return redirect(url_for("home"))


@articles_bp.route('/history', methods=['GET'])
@articles_bp.route('/history/<int:year>', methods=['GET'])
@articles_bp.route('/history/<int:year>/<int:month>', methods=['GET'])
@articles_bp.route("/history", methods=["GET"])
@articles_bp.route("/history/<int:year>", methods=["GET"])
@articles_bp.route("/history/<int:year>/<int:month>", methods=["GET"])
@login_required
def history(year=None, month=None):
    cntr, artcles = ArticleController(current_user.id).get_history(year, month)
    return render_template('history.html', articles_counter=cntr,
                           articles=artcles, year=year, month=month)
    return render_template(
        "history.html", articles_counter=cntr, articles=artcles, year=year, month=month
    )


@article_bp.route('/mark_as/<string:new_value>', methods=['GET'])
@article_bp.route('/mark_as/<string:new_value>/article/<int:article_id>',
                  methods=['GET'])
@article_bp.route("/mark_as/<string:new_value>", methods=["GET"])
@article_bp.route(
    "/mark_as/<string:new_value>/article/<int:article_id>", methods=["GET"]
)
@login_required
def mark_as(new_value='read', feed_id=None, article_id=None):
def mark_as(new_value="read", feed_id=None, article_id=None):
    """
    Mark all unreaded articles as read.
    """
    readed = new_value == 'read'
    readed = new_value == "read"
    art_contr = ArticleController(current_user.id)
    filters = {'readed': not readed}
    filters = {"readed": not readed}
    if feed_id is not None:
        filters['feed_id'] = feed_id
        message = 'Feed marked as %s.'
        filters["feed_id"] = feed_id
        message = "Feed marked as %s."
    elif article_id is not None:
        filters['id'] = article_id
        message = 'Article marked as %s.'
        filters["id"] = article_id
        message = "Article marked as %s."
    else:
        message = 'All article marked as %s.'
        message = "All article marked as %s."
    art_contr.update(filters, {"readed": readed})
    flash(gettext(message % new_value), 'info')
    flash(gettext(message % new_value), "info")

    if readed:
        return redirect(redirect_url())
    return redirect('home')
    return redirect("home")


@articles_bp.route('/expire_articles', methods=['GET'])
@articles_bp.route("/expire_articles", methods=["GET"])
@login_required
def expire():
    """
    Delete articles older than the given number of weeks.
    """
    current_time = datetime.utcnow()
    weeks_ago = current_time - timedelta(int(request.args.get('weeks', 10)))
    weeks_ago = current_time - timedelta(int(request.args.get("weeks", 10)))
    art_contr = ArticleController(current_user.id)

    query = art_contr.read(__or__={'date__lt': weeks_ago,
                                   'retrieved_date__lt': weeks_ago})
    query = art_contr.read(
        __or__={"date__lt": weeks_ago, "retrieved_date__lt": weeks_ago}
    )
    count = query.count()
    query.delete()
    db.session.commit()
    flash(gettext('%(count)d articles deleted', count=count), 'info')
    flash(gettext("%(count)d articles deleted", count=count), "info")
    return redirect(redirect_url())


@articles_bp.route('/export', methods=['GET'])
@articles_bp.route("/export", methods=["GET"])
@login_required
def export():
    """


@@ 145,10 157,9 @@ def export():
    try:
        json_result = export_json(user)
    except Exception as e:
        flash(gettext("Error when exporting articles."), 'danger')
        flash(gettext("Error when exporting articles."), "danger")
        return redirect(redirect_url())
    response = make_response(json_result)
    response.mimetype = 'application/json'
    response.headers["Content-Disposition"] \
            = 'attachment; filename=account.json'
    response.mimetype = "application/json"
    response.headers["Content-Disposition"] = "attachment; filename=account.json"
    return response

M newspipe/web/views/bookmark.py => newspipe/web/views/bookmark.py +140 -106
@@ 1,5 1,5 @@
#! /usr/bin/env python
#-*- coding: utf-8 -*-
# -*- coding: utf-8 -*-

# Newspipe - A Web based news aggregator.
# Copyright (C) 2010-2017  Cédric Bonhomme - https://www.cedricbonhomme.org


@@ 30,8 30,15 @@ import logging
import datetime
from werkzeug.exceptions import BadRequest

from flask import Blueprint, render_template, flash, \
                  redirect, request, url_for, make_response
from flask import (
    Blueprint,
    render_template,
    flash,
    redirect,
    request,
    url_for,
    make_response,
)
from flask_babel import gettext
from flask_login import login_required, current_user
from flask_paginate import Pagination, get_page_args


@@ 46,93 53,105 @@ from web.controllers import BookmarkController, BookmarkTagController
from web.models import BookmarkTag

logger = logging.getLogger(__name__)
bookmarks_bp = Blueprint('bookmarks', __name__, url_prefix='/bookmarks')
bookmark_bp = Blueprint('bookmark', __name__, url_prefix='/bookmark')
bookmarks_bp = Blueprint("bookmarks", __name__, url_prefix="/bookmarks")
bookmark_bp = Blueprint("bookmark", __name__, url_prefix="/bookmark")


@bookmarks_bp.route('/', defaults={'per_page': '50'}, methods=['GET'])
@bookmarks_bp.route('/<string:status>', defaults={'per_page': '50'},
                                                                methods=['GET'])
def list_(per_page, status='all'):
@bookmarks_bp.route("/", defaults={"per_page": "50"}, methods=["GET"])
@bookmarks_bp.route("/<string:status>", defaults={"per_page": "50"}, methods=["GET"])
def list_(per_page, status="all"):
    "Lists the bookmarks."
    head_titles = [gettext("Bookmarks")]
    user_id = None
    filters = {}
    tag = request.args.get('tag', None)
    tag = request.args.get("tag", None)
    if tag:
        filters['tags_proxy__contains'] = tag
    query = request.args.get('query', None)
        filters["tags_proxy__contains"] = tag
    query = request.args.get("query", None)
    if query:
        query_regex = '%' + query + '%'
        filters['__or__'] = {'title__ilike': query_regex,
                            'description__ilike': query_regex}
        query_regex = "%" + query + "%"
        filters["__or__"] = {
            "title__ilike": query_regex,
            "description__ilike": query_regex,
        }
    if current_user.is_authenticated:
        # query for the bookmarks of the authenticated user
        user_id = current_user.id
        if status == 'public':
        if status == "public":
            # load public bookmarks only
            filters['shared'] = True
        elif status == 'private':
            filters["shared"] = True
        elif status == "private":
            # load private bookmarks only
            filters['shared'] = False
            filters["shared"] = False
        else:
            # no filter: load shared and public bookmarks
            pass
        if status == 'unread':
            filters['to_read'] = True
        if status == "unread":
            filters["to_read"] = True
        else:
            pass
    else:
        # query for the shared bookmarks (of all users)
        head_titles = [gettext("Recent bookmarks")]
        not_created_before = datetime.datetime.today() - \
                                                    datetime.timedelta(days=900)
        filters['time__gt'] = not_created_before # only "recent" bookmarks
        filters['shared'] = True
        not_created_before = datetime.datetime.today() - datetime.timedelta(days=900)
        filters["time__gt"] = not_created_before  # only "recent" bookmarks
        filters["shared"] = True

    bookmarks = BookmarkController(user_id) \
                    .read(**filters) \
                    .order_by(desc('time'))
    bookmarks = BookmarkController(user_id).read(**filters).order_by(desc("time"))

    #tag_contr = BookmarkTagController(user_id)
    #tag_contr.read().join(bookmarks).all()
    # tag_contr = BookmarkTagController(user_id)
    # tag_contr.read().join(bookmarks).all()

    page, per_page, offset = get_page_args()
    pagination = Pagination(page=page, total=bookmarks.count(),
                            css_framework='bootstrap3',
                            search=False, record_name='bookmarks',
                            per_page=per_page)

    return render_template('bookmarks.html',
                            head_titles=head_titles,
                            bookmarks=bookmarks.offset(offset).limit(per_page),
                            pagination=pagination,
                            tag=tag,
                            query=query)


@bookmark_bp.route('/create', methods=['GET'])
@bookmark_bp.route('/edit/<int:bookmark_id>', methods=['GET'])
    pagination = Pagination(
        page=page,
        total=bookmarks.count(),
        css_framework="bootstrap3",
        search=False,
        record_name="bookmarks",
        per_page=per_page,
    )

    return render_template(
        "bookmarks.html",
        head_titles=head_titles,
        bookmarks=bookmarks.offset(offset).limit(per_page),
        pagination=pagination,
        tag=tag,
        query=query,
    )


@bookmark_bp.route("/create", methods=["GET"])
@bookmark_bp.route("/edit/<int:bookmark_id>", methods=["GET"])
@login_required
def form(bookmark_id=None):
    "Form to create/edit bookmarks."
    action = gettext("Add a new bookmark")
    head_titles = [action]
    if bookmark_id is None:
        return render_template('edit_bookmark.html', action=action,
                               head_titles=head_titles, form=BookmarkForm())
        return render_template(
            "edit_bookmark.html",
            action=action,
            head_titles=head_titles,
            form=BookmarkForm(),
        )
    bookmark = BookmarkController(current_user.id).get(id=bookmark_id)
    action = gettext('Edit bookmark')
    action = gettext("Edit bookmark")
    head_titles = [action]
    form = BookmarkForm(obj=bookmark)
    form.tags.data = ", ".join(bookmark.tags_proxy)
    return render_template('edit_bookmark.html', action=action,
                           head_titles=head_titles, bookmark=bookmark,
                           form=form)
    return render_template(
        "edit_bookmark.html",
        action=action,
        head_titles=head_titles,
        bookmark=bookmark,
        form=form,
    )


@bookmark_bp.route('/create', methods=['POST'])
@bookmark_bp.route('/edit/<int:bookmark_id>', methods=['POST'])
@bookmark_bp.route("/create", methods=["POST"])
@bookmark_bp.route("/edit/<int:bookmark_id>", methods=["POST"])
@login_required
def process_form(bookmark_id=None):
    "Process the creation/edition of bookmarks."


@@ 141,116 160,131 @@ def process_form(bookmark_id=None):
    tag_contr = BookmarkTagController(current_user.id)

    if not form.validate():
        return render_template('edit_bookmark.html', form=form)
        return render_template("edit_bookmark.html", form=form)

    if form.title.data == '':
    if form.title.data == "":
        title = form.href.data
    else:
        title = form.title.data

    bookmark_attr = {'href': form.href.data,
                    'description': form.description.data,
                    'title': title,
                    'shared': form.shared.data,
                    'to_read': form.to_read.data}
    bookmark_attr = {
        "href": form.href.data,
        "description": form.description.data,
        "title": title,
        "shared": form.shared.data,
        "to_read": form.to_read.data,
    }

    if bookmark_id is not None:
        tags = []
        for tag in form.tags.data.split(','):
            new_tag = tag_contr.create(text=tag.strip(), user_id=current_user.id,
                                        bookmark_id=bookmark_id)
        for tag in form.tags.data.split(","):
            new_tag = tag_contr.create(
                text=tag.strip(), user_id=current_user.id, bookmark_id=bookmark_id
            )
            tags.append(new_tag)
        bookmark_attr['tags'] = tags
        bookmark_contr.update({'id': bookmark_id}, bookmark_attr)
        flash(gettext('Bookmark successfully updated.'), 'success')
        return redirect(url_for('bookmark.form', bookmark_id=bookmark_id))
        bookmark_attr["tags"] = tags
        bookmark_contr.update({"id": bookmark_id}, bookmark_attr)
        flash(gettext("Bookmark successfully updated."), "success")
        return redirect(url_for("bookmark.form", bookmark_id=bookmark_id))

    # Create a new bookmark
    new_bookmark = bookmark_contr.create(**bookmark_attr)
    tags = []
    for tag in form.tags.data.split(','):
        new_tag = tag_contr.create(text=tag.strip(), user_id=current_user.id,
                                    bookmark_id=new_bookmark.id)
    for tag in form.tags.data.split(","):
        new_tag = tag_contr.create(
            text=tag.strip(), user_id=current_user.id, bookmark_id=new_bookmark.id
        )
        tags.append(new_tag)
    bookmark_attr['tags'] = tags
    bookmark_contr.update({'id': new_bookmark.id}, bookmark_attr)
    flash(gettext('Bookmark successfully created.'), 'success')
    return redirect(url_for('bookmark.form', bookmark_id=new_bookmark.id))
    bookmark_attr["tags"] = tags
    bookmark_contr.update({"id": new_bookmark.id}, bookmark_attr)
    flash(gettext("Bookmark successfully created."), "success")
    return redirect(url_for("bookmark.form", bookmark_id=new_bookmark.id))


@bookmark_bp.route('/delete/<int:bookmark_id>', methods=['GET'])
@bookmark_bp.route("/delete/<int:bookmark_id>", methods=["GET"])
@login_required
def delete(bookmark_id=None):
    "Delete a bookmark."
    bookmark = BookmarkController(current_user.id).delete(bookmark_id)
    flash(gettext("Bookmark %(bookmark_name)s successfully deleted.",
                  bookmark_name=bookmark.title), 'success')
    return redirect(url_for('bookmarks.list_'))
    flash(
        gettext(
            "Bookmark %(bookmark_name)s successfully deleted.",
            bookmark_name=bookmark.title,
        ),
        "success",
    )
    return redirect(url_for("bookmarks.list_"))


@bookmarks_bp.route('/delete', methods=['GET'])
@bookmarks_bp.route("/delete", methods=["GET"])
@login_required
def delete_all():
    "Delete all bookmarks."
    bookmark = BookmarkController(current_user.id).read().delete()
    db.session.commit()
    flash(gettext("Bookmarks successfully deleted."), 'success')
    flash(gettext("Bookmarks successfully deleted."), "success")
    return redirect(redirect_url())


@bookmark_bp.route('/bookmarklet', methods=['GET', 'POST'])
@bookmark_bp.route("/bookmarklet", methods=["GET", "POST"])
@login_required
def bookmarklet():
    bookmark_contr = BookmarkController(current_user.id)
    href = (request.args if request.method == 'GET' else request.form)\
            .get('href', None)
    href = (request.args if request.method == "GET" else request.form).get("href", None)
    if not href:
        flash(gettext("Couldn't add bookmark: url missing."), "error")
        raise BadRequest("url is missing")
    title = (request.args if request.method == 'GET' else request.form)\
            .get('title', None)
    title = (request.args if request.method == "GET" else request.form).get(
        "title", None
    )
    if not title:
        title = href

    bookmark_exists = bookmark_contr.read(**{'href': href}).all()
    bookmark_exists = bookmark_contr.read(**{"href": href}).all()
    if bookmark_exists:
        flash(gettext("Couldn't add bookmark: bookmark already exists."),
                "warning")
        return redirect(url_for('bookmark.form',
                                            bookmark_id=bookmark_exists[0].id))
        flash(gettext("Couldn't add bookmark: bookmark already exists."), "warning")
        return redirect(url_for("bookmark.form", bookmark_id=bookmark_exists[0].id))

    bookmark_attr = {'href': href,
                    'description': '',
                    'title': title,
                    'shared': True,
                    'to_read': True}
    bookmark_attr = {
        "href": href,
        "description": "",
        "title": title,
        "shared": True,
        "to_read": True,
    }

    new_bookmark = bookmark_contr.create(**bookmark_attr)
    flash(gettext('Bookmark successfully created.'), 'success')
    return redirect(url_for('bookmark.form', bookmark_id=new_bookmark.id))
    flash(gettext("Bookmark successfully created."), "success")
    return redirect(url_for("bookmark.form", bookmark_id=new_bookmark.id))


@bookmark_bp.route('/import_pinboard', methods=['POST'])
@bookmark_bp.route("/import_pinboard", methods=["POST"])
@login_required
def import_pinboard():
    bookmarks = request.files.get('jsonfile', None)
    bookmarks = request.files.get("jsonfile", None)
    if bookmarks:
        try:
            nb_bookmarks = import_pinboard_json(current_user, bookmarks.read())
            flash(gettext("%(nb_bookmarks)s bookmarks successfully imported.",
                          nb_bookmarks=nb_bookmarks), 'success')
            flash(
                gettext(
                    "%(nb_bookmarks)s bookmarks successfully imported.",
                    nb_bookmarks=nb_bookmarks,
                ),
                "success",
            )
        except Exception as e:
            flash(gettext('Error when importing bookmarks.'), 'error')
            flash(gettext("Error when importing bookmarks."), "error")

    return redirect(redirect_url())


@bookmarks_bp.route('/export', methods=['GET'])
@bookmarks_bp.route("/export", methods=["GET"])
@login_required
def export():
    bookmarks = export_bookmarks(current_user)
    response = make_response(bookmarks)
    response.mimetype = 'application/json'
    response.headers["Content-Disposition"] \
            = 'attachment; filename=newspipe_bookmarks_export.json'
    response.mimetype = "application/json"
    response.headers[
        "Content-Disposition"
    ] = "attachment; filename=newspipe_bookmarks_export.json"
    return response

M newspipe/web/views/category.py => newspipe/web/views/category.py +57 -34
@@ 5,82 5,105 @@ from flask_login import login_required, current_user
from web.forms import CategoryForm
from lib.utils import redirect_url
from web.lib.view_utils import etag_match
from web.controllers import ArticleController, FeedController, \
                            CategoryController
from web.controllers import ArticleController, FeedController, CategoryController

categories_bp = Blueprint('categories', __name__, url_prefix='/categories')
category_bp = Blueprint('category', __name__, url_prefix='/category')
categories_bp = Blueprint("categories", __name__, url_prefix="/categories")
category_bp = Blueprint("category", __name__, url_prefix="/category")


@categories_bp.route('/', methods=['GET'])
@categories_bp.route("/", methods=["GET"])
@login_required
@etag_match
def list_():
    "Lists the subscribed feeds in a table."
    art_contr = ArticleController(current_user.id)
    return render_template('categories.html',
            categories=list(CategoryController(current_user.id).read().order_by('name')),
            feeds_count=FeedController(current_user.id).count_by_category(),
            unread_article_count=art_contr.count_by_category(readed=False),
            article_count=art_contr.count_by_category())
    return render_template(
        "categories.html",
        categories=list(CategoryController(current_user.id).read().order_by("name")),
        feeds_count=FeedController(current_user.id).count_by_category(),
        unread_article_count=art_contr.count_by_category(readed=False),
        article_count=art_contr.count_by_category(),
    )


@category_bp.route('/create', methods=['GET'])
@category_bp.route('/edit/<int:category_id>', methods=['GET'])
@category_bp.route("/create", methods=["GET"])
@category_bp.route("/edit/<int:category_id>", methods=["GET"])
@login_required
@etag_match
def form(category_id=None):
    action = gettext("Add a category")
    head_titles = [action]
    if category_id is None:
        return render_template('edit_category.html', action=action,
                               head_titles=head_titles, form=CategoryForm())
        return render_template(
            "edit_category.html",
            action=action,
            head_titles=head_titles,
            form=CategoryForm(),
        )
    category = CategoryController(current_user.id).get(id=category_id)
    action = gettext('Edit category')
    action = gettext("Edit category")
    head_titles = [action]
    if category.name:
        head_titles.append(category.name)
    return render_template('edit_category.html', action=action,
                           head_titles=head_titles, category=category,
                           form=CategoryForm(obj=category))
    return render_template(
        "edit_category.html",
        action=action,
        head_titles=head_titles,
        category=category,
        form=CategoryForm(obj=category),
    )


@category_bp.route('/delete/<int:category_id>', methods=['GET'])
@category_bp.route("/delete/<int:category_id>", methods=["GET"])
@login_required
def delete(category_id=None):
    category = CategoryController(current_user.id).delete(category_id)
    flash(gettext("Category %(category_name)s successfully deleted.",
                  category_name=category.name), 'success')
    flash(
        gettext(
            "Category %(category_name)s successfully deleted.",
            category_name=category.name,
        ),
        "success",
    )
    return redirect(redirect_url())


@category_bp.route('/create', methods=['POST'])
@category_bp.route('/edit/<int:category_id>', methods=['POST'])
@category_bp.route("/create", methods=["POST"])
@category_bp.route("/edit/<int:category_id>", methods=["POST"])
@login_required
def process_form(category_id=None):
    form = CategoryForm()
    cat_contr = CategoryController(current_user.id)

    if not form.validate():
        return render_template('edit_category.html', form=form)
        return render_template("edit_category.html", form=form)
    existing_cats = list(cat_contr.read(name=form.name.data))
    if existing_cats and category_id is None:
        flash(gettext("Couldn't add category: already exists."), "warning")
        return redirect(url_for('category.form',
                                category_id=existing_cats[0].id))
        return redirect(url_for("category.form", category_id=existing_cats[0].id))
    # Edit an existing category
    category_attr = {'name': form.name.data}
    category_attr = {"name": form.name.data}

    if category_id is not None:
        cat_contr.update({'id': category_id}, category_attr)
        flash(gettext('Category %(cat_name)r successfully updated.',
                      cat_name=category_attr['name']), 'success')
        return redirect(url_for('category.form', category_id=category_id))
        cat_contr.update({"id": category_id}, category_attr)
        flash(
            gettext(
                "Category %(cat_name)r successfully updated.",
                cat_name=category_attr["name"],
            ),
            "success",
        )
        return redirect(url_for("category.form", category_id=category_id))

    # Create a new category
    new_category = cat_contr.create(**category_attr)

    flash(gettext('Category %(category_name)r successfully created.',
                  category_name=new_category.name), 'success')
    flash(
        gettext(
            "Category %(category_name)r successfully created.",
            category_name=new_category.name,
        ),
        "success",
    )

    return redirect(url_for('category.form', category_id=new_category.id))
    return redirect(url_for("category.form", category_id=new_category.id))

M newspipe/web/views/common.py => newspipe/web/views/common.py +21 -11
@@ 3,13 3,18 @@ from functools import wraps
from datetime import datetime
from flask import current_app, Response
from flask_login import login_user
from flask_principal import (Identity, Permission, RoleNeed,
                                 session_identity_loader, identity_changed)
from flask_principal import (
    Identity,
    Permission,
    RoleNeed,
    session_identity_loader,
    identity_changed,
)
from web.controllers import UserController
from lib.utils import default_handler

admin_role = RoleNeed('admin')
api_role = RoleNeed('api')
admin_role = RoleNeed("admin")
api_role = RoleNeed("api")

admin_permission = Permission(admin_role)
api_permission = Permission(api_role)


@@ 17,21 22,23 @@ api_permission = Permission(api_role)

def scoped_default_handler():
    if admin_permission.can():
        role = 'admin'
        role = "admin"
    elif api_permission.can():
        role = 'api'
        role = "api"
    else:
        role = 'user'
        role = "user"

    @wraps(default_handler)
    def wrapper(obj):
        return default_handler(obj, role=role)

    return wrapper


def jsonify(func):
    """Will cast results of func as a result, and try to extract
    a status_code for the Response object"""

    @wraps(func)
    def wrapper(*args, **kwargs):
        status_code = 200


@@ 40,8 47,12 @@ def jsonify(func):
            return result
        elif isinstance(result, tuple):
            result, status_code = result
        return Response(json.dumps(result, default=scoped_default_handler()),
                        mimetype='application/json', status=status_code)
        return Response(
            json.dumps(result, default=scoped_default_handler()),
            mimetype="application/json",
            status=status_code,
        )

    return wrapper




@@ 49,5 60,4 @@ def login_user_bundle(user):
    login_user(user)
    identity_changed.send(current_app, identity=Identity(user.id))
    session_identity_loader()
    UserController(user.id).update(
                {'id': user.id}, {'last_seen': datetime.utcnow()})
    UserController(user.id).update({"id": user.id}, {"last_seen": datetime.utcnow()})

M newspipe/web/views/feed.py => newspipe/web/views/feed.py +179 -130
@@ 4,8 4,15 @@ from datetime import datetime, timedelta
from sqlalchemy import desc
from werkzeug.exceptions import BadRequest

from flask import Blueprint, render_template, flash, \
                  redirect, request, url_for, make_response
from flask import (
    Blueprint,
    render_template,
    flash,
    redirect,
    request,
    url_for,
    make_response,
)
from flask_babel import gettext
from flask_login import login_required, current_user
from flask_paginate import Pagination, get_page_args


@@ 15,24 22,30 @@ from lib import misc_utils, utils
from lib.feed_utils import construct_feed_from
from web.lib.view_utils import etag_match
from web.forms import AddFeedForm
from web.controllers import (UserController, CategoryController,
                                FeedController, ArticleController)
from web.controllers import (
    UserController,
    CategoryController,
    FeedController,
    ArticleController,
)

logger = logging.getLogger(__name__)
feeds_bp = Blueprint('feeds', __name__, url_prefix='/feeds')
feed_bp = Blueprint('feed', __name__, url_prefix='/feed')
feeds_bp = Blueprint("feeds", __name__, url_prefix="/feeds")
feed_bp = Blueprint("feed", __name__, url_prefix="/feed")


@feeds_bp.route('/', methods=['GET'])
@feeds_bp.route("/", methods=["GET"])
@login_required
@etag_match
def feeds():
    "Lists the subscribed feeds in a table."
    art_contr = ArticleController(current_user.id)
    return render_template('feeds.html',
            feeds=FeedController(current_user.id).read().order_by('title'),
            unread_article_count=art_contr.count_by_feed(readed=False),
            article_count=art_contr.count_by_feed())
    return render_template(
        "feeds.html",
        feeds=FeedController(current_user.id).read().order_by("title"),
        unread_article_count=art_contr.count_by_feed(readed=False),
        article_count=art_contr.count_by_feed(),
    )


def feed_view(feed_id=None, user_id=None):


@@ 42,15 55,19 @@ def feed_view(feed_id=None, user_id=None):
    if feed.category_id:
        category = CategoryController(user_id).get(id=feed.category_id)
    filters = {}
    filters['feed_id'] = feed_id
    filters["feed_id"] = feed_id
    articles = ArticleController(user_id).read_light(**filters)

    # Server-side pagination
    page, per_page, offset = get_page_args(per_page_parameter='per_page')
    pagination = Pagination(page=page, total=articles.count(),
                            css_framework='bootstrap3',
                            search=False, record_name='articles',
                            per_page=per_page)
    page, per_page, offset = get_page_args(per_page_parameter="per_page")
    pagination = Pagination(
        page=page,
        total=articles.count(),
        css_framework="bootstrap3",
        search=False,
        record_name="articles",
        per_page=per_page,
    )

    today = datetime.now()
    try:


@@ 65,17 82,22 @@ def feed_view(feed_id=None, user_id=None):
        average = 0
    elapsed = today - last_article

    return render_template('feed.html',
                           head_titles=[utils.clear_string(feed.title)],
                           feed=feed, category=category,
                           articles=articles.offset(offset).limit(per_page),
                           pagination=pagination,
                           first_post_date=first_article,
                           end_post_date=last_article,
                           average=average, delta=delta, elapsed=elapsed)


@feed_bp.route('/<int:feed_id>', methods=['GET'])
    return render_template(
        "feed.html",
        head_titles=[utils.clear_string(feed.title)],
        feed=feed,
        category=category,
        articles=articles.offset(offset).limit(per_page),
        pagination=pagination,
        first_post_date=first_article,
        end_post_date=last_article,
        average=average,
        delta=delta,
        elapsed=elapsed,
    )


@feed_bp.route("/<int:feed_id>", methods=["GET"])
@login_required
@etag_match
def feed(feed_id=None):


@@ 83,7 105,7 @@ def feed(feed_id=None):
    return feed_view(feed_id, current_user.id)


@feed_bp.route('/public/<int:feed_id>', methods=['GET'])
@feed_bp.route("/public/<int:feed_id>", methods=["GET"])
@etag_match
def feed_pub(feed_id=None):
    """


@@ 92,90 114,97 @@ def feed_pub(feed_id=None):
    """
    feed = FeedController(None).get(id=feed_id)
    if feed.private or not feed.user.is_public_profile:
        return render_template('errors/404.html'), 404
        return render_template("errors/404.html"), 404
    return feed_view(feed_id, None)


@feed_bp.route('/delete/<feed_id>', methods=['GET'])
@feed_bp.route("/delete/<feed_id>", methods=["GET"])
@login_required
def delete(feed_id=None):
    feed_contr = FeedController(current_user.id)
    feed = feed_contr.get(id=feed_id)
    feed_contr.delete(feed_id)
    flash(gettext("Feed %(feed_title)s successfully deleted.",
                  feed_title=feed.title), 'success')
    return redirect(url_for('home'))
    flash(
        gettext("Feed %(feed_title)s successfully deleted.", feed_title=feed.title),
        "success",
    )
    return redirect(url_for("home"))


@feed_bp.route('/reset_errors/<int:feed_id>', methods=['GET', 'POST'])
@feed_bp.route("/reset_errors/<int:feed_id>", methods=["GET", "POST"])
@login_required
def reset_errors(feed_id):
    feed_contr = FeedController(current_user.id)
    feed = feed_contr.get(id=feed_id)
    feed_contr.update({'id': feed_id}, {'error_count': 0, 'last_error': ''})
    flash(gettext('Feed %(feed_title)r successfully updated.',
                  feed_title=feed.title), 'success')
    return redirect(request.referrer or url_for('home'))
    feed_contr.update({"id": feed_id}, {"error_count": 0, "last_error": ""})
    flash(
        gettext("Feed %(feed_title)r successfully updated.", feed_title=feed.title),
        "success",
    )
    return redirect(request.referrer or url_for("home"))


@feed_bp.route('/bookmarklet', methods=['GET', 'POST'])
@feed_bp.route("/bookmarklet", methods=["GET", "POST"])
@login_required
def bookmarklet():
    feed_contr = FeedController(current_user.id)
    url = (request.args if request.method == 'GET' else request.form)\
            .get('url', None)
    url = (request.args if request.method == "GET" else request.form).get("url", None)
    if not url:
        flash(gettext("Couldn't add feed: url missing."), "error")
        raise BadRequest("url is missing")

    feed_exists = list(feed_contr.read(__or__={'link': url, 'site_link': url}))
    feed_exists = list(feed_contr.read(__or__={"link": url, "site_link": url}))
    if feed_exists:
        flash(gettext("Couldn't add feed: feed already exists."),
                "warning")
        return redirect(url_for('feed.form', feed_id=feed_exists[0].id))
        flash(gettext("Couldn't add feed: feed already exists."), "warning")
        return redirect(url_for("feed.form", feed_id=feed_exists[0].id))

    try:
        feed = construct_feed_from(url)
    except requests.exceptions.ConnectionError:
        flash(gettext("Impossible to connect to the address: {}.".format(url)),
              "danger")
        return redirect(url_for('home'))
        flash(
            gettext("Impossible to connect to the address: {}.".format(url)), "danger"
        )
        return redirect(url_for("home"))
    except Exception:
        logger.exception('something bad happened when fetching %r', url)
        return redirect(url_for('home'))
    if not feed.get('link'):
        feed['enabled'] = False
        flash(gettext("Couldn't find a feed url, you'll need to find a Atom or"
                      " RSS link manually and reactivate this feed"),
              'warning')
        logger.exception("something bad happened when fetching %r", url)
        return redirect(url_for("home"))
    if not feed.get("link"):
        feed["enabled"] = False
        flash(
            gettext(
                "Couldn't find a feed url, you'll need to find a Atom or"
                " RSS link manually and reactivate this feed"
            ),
            "warning",
        )
    feed = feed_contr.create(**feed)
    flash(gettext('Feed was successfully created.'), 'success')
    flash(gettext("Feed was successfully created."), "success")
    if feed.enabled and conf.CRAWLING_METHOD == "default":
        misc_utils.fetch(current_user.id, feed.id)
        flash(gettext("Downloading articles for the new feed..."), 'info')
    return redirect(url_for('feed.form', feed_id=feed.id))
        flash(gettext("Downloading articles for the new feed..."), "info")
    return redirect(url_for("feed.form", feed_id=feed.id))


@feed_bp.route('/update/<action>/<int:feed_id>', methods=['GET', 'POST'])
@feeds_bp.route('/update/<action>', methods=['GET', 'POST'])
@feed_bp.route("/update/<action>/<int:feed_id>", methods=["GET", "POST"])
@feeds_bp.route("/update/<action>", methods=["GET", "POST"])
@login_required
def update(action, feed_id=None):
    readed = action == 'read'
    filters = {'readed__ne': readed}
    readed = action == "read"
    filters = {"readed__ne": readed}

    nb_days = request.args.get('nb_days', 0, type=int)
    nb_days = request.args.get("nb_days", 0, type=int)
    if nb_days != 0:
        filters['date__lt'] = datetime.now() - timedelta(days=nb_days)
        filters["date__lt"] = datetime.now() - timedelta(days=nb_days)

    if feed_id:
        filters['feed_id'] = feed_id
    ArticleController(current_user.id).update(filters, {'readed': readed})
    flash(gettext('Feed successfully updated.'), 'success')
    return redirect(request.referrer or url_for('home'))
        filters["feed_id"] = feed_id
    ArticleController(current_user.id).update(filters, {"readed": readed})
    flash(gettext("Feed successfully updated."), "success")
    return redirect(request.referrer or url_for("home"))


@feed_bp.route('/create', methods=['GET'])
@feed_bp.route('/edit/<int:feed_id>', methods=['GET'])
@feed_bp.route("/create", methods=["GET"])
@feed_bp.route("/edit/<int:feed_id>", methods=["GET"])
@login_required
@etag_match
def form(feed_id=None):


@@ 185,22 214,28 @@ def form(feed_id=None):
    if feed_id is None:
        form = AddFeedForm()
        form.set_category_choices(categories)
        return render_template('edit_feed.html', action=action,
                               head_titles=head_titles, form=form)
        return render_template(
            "edit_feed.html", action=action, head_titles=head_titles, form=form
        )
    feed = FeedController(current_user.id).get(id=feed_id)
    form = AddFeedForm(obj=feed)
    form.set_category_choices(categories)
    action = gettext('Edit feed')
    action = gettext("Edit feed")
    head_titles = [action]
    if feed.title:
        head_titles.append(feed.title)
    return render_template('edit_feed.html', action=action,
                           head_titles=head_titles, categories=categories,
                           form=form, feed=feed)


@feed_bp.route('/create', methods=['POST'])
@feed_bp.route('/edit/<int:feed_id>', methods=['POST'])
    return render_template(
        "edit_feed.html",
        action=action,
        head_titles=head_titles,
        categories=categories,
        form=form,
        feed=feed,
    )


@feed_bp.route("/create", methods=["POST"])
@feed_bp.route("/edit/<int:feed_id>", methods=["POST"])
@login_required
def process_form(feed_id=None):
    form = AddFeedForm()


@@ 208,58 243,68 @@ def process_form(feed_id=None):
    form.set_category_choices(CategoryController(current_user.id).read())

    if not form.validate():
        return render_template('edit_feed.html', form=form)
        return render_template("edit_feed.html", form=form)
    existing_feeds = list(feed_contr.read(link=form.link.data))
    if existing_feeds and feed_id is None:
        flash(gettext("Couldn't add feed: feed already exists."), "warning")
        return redirect(url_for('feed.form', feed_id=existing_feeds[0].id))
        return redirect(url_for("feed.form", feed_id=existing_feeds[0].id))
    # Edit an existing feed
    feed_attr = {'title': form.title.data, 'enabled': form.enabled.data,
                 'link': form.link.data, 'site_link': form.site_link.data,
                 'filters': [], 'category_id': form.category_id.data,
                 'private': form.private.data}
    if not feed_attr['category_id'] or feed_attr['category_id'] == '0':
        del feed_attr['category_id']

    for filter_attr in ('type', 'pattern', 'action on', 'action'):
        for i, value in enumerate(
                request.form.getlist(filter_attr.replace(' ', '_'))):
            if i >= len(feed_attr['filters']):
                feed_attr['filters'].append({})
            feed_attr['filters'][i][filter_attr] = value
    feed_attr = {
        "title": form.title.data,
        "enabled": form.enabled.data,
        "link": form.link.data,
        "site_link": form.site_link.data,