~cedric/newspipe

119a8f3e7c2dc0840312ee9e051b899e7b2c031d — Cédric Bonhomme 2 months ago 9c0c7b6
deleted read article (and not liked) that are retrieved since more than 15 days.
M newspipe/bootstrap.py => newspipe/bootstrap.py +3 -1
@@ 69,7 69,9 @@ db = SQLAlchemy(application)

migrate = Migrate(application, db)

talisman = Talisman(application, content_security_policy=application.config["CONTENT_SECURITY_POLICY"])
talisman = Talisman(
    application, content_security_policy=application.config["CONTENT_SECURITY_POLICY"]
)

babel = Babel(application)


M newspipe/commands.py => newspipe/commands.py +10 -9
@@ 34,8 34,8 @@ def db_create():


@application.cli.command("create_admin")
@click.option('--nickname', default='admin', help='Nickname')
@click.option('--password', default='password', help='Password')
@click.option("--nickname", default="admin", help="Nickname")
@click.option("--password", default="password", help="Password")
def create_admin(nickname, password):
    "Will create an admin user."
    admin = {


@@ 53,7 53,7 @@ def create_admin(nickname, password):


@application.cli.command("delete_user")
@click.option('--user-id', required=True, help='Id of the user to delete.')
@click.option("--user-id", required=True, help="Id of the user to delete.")
def delete_user(user_id=None):
    "Delete the user with the id specified in the command line."
    try:


@@ 64,7 64,7 @@ def delete_user(user_id=None):


@application.cli.command("delete_inactive_users")
@click.option('--last-seen', default=6, help='Number of months since last seen.')
@click.option("--last-seen", default=6, help="Number of months since last seen.")
def delete_inactive_users(last_seen):
    "Delete inactive users (inactivity is given in parameter and specified in number of months)."
    filter = {}


@@ 81,7 81,7 @@ def delete_inactive_users(last_seen):


@application.cli.command("disable_inactive_users")
@click.option('--last-seen', default=6, help='Number of months since last seen.')
@click.option("--last-seen", default=6, help="Number of months since last seen.")
def disable_inactive_users(last_seen):
    "Disable inactive users (inactivity is given in parameter and specified in number of months)."
    filter = {}


@@ 101,10 101,11 @@ def disable_inactive_users(last_seen):

@application.cli.command("delete_read_articles")
def delete_read_articles():
    "Delete read articles retrieved since more than 15 days ago."
    "Delete read articles (and not liked) retrieved since more than 15 days ago."
    filter = {}
    filter["user_id__ne"] = 1
    #filter["readed"] = True # temporary comment
    filter["readed"] = True
    filter["like"] = False
    filter["retrieved_date__lt"] = date.today() - relativedelta(days=15)
    articles = ArticleController().read(**filter).limit(5000)
    for article in articles:


@@ 130,8 131,8 @@ def fix_article_entry_id():


@application.cli.command("fetch_asyncio")
@click.option('--user-id', default=None, help='Id of the user')
@click.option('--feed-id', default=None, help='If of the feed')
@click.option("--user-id", default=None, help="Id of the user")
@click.option("--feed-id", default=None, help="If of the feed")
def fetch_asyncio(user_id=None, feed_id=None):
    "Crawl the feeds with asyncio."
    import asyncio

M newspipe/controllers/abstract.py => newspipe/controllers/abstract.py +1 -1
@@ 63,7 63,7 @@ class AbstractController:
        return db_filters

    def _get(self, **filters):
        """ Will add the current user id if that one is not none (in which case
        """Will add the current user id if that one is not none (in which case
        the decision has been made in the code that the query shouldn't be user
        dependent) and the user is not an admin and the filters doesn't already
        contains a filter for that user.

M newspipe/controllers/feed.py => newspipe/controllers/feed.py +1 -3
@@ 88,9 88,7 @@ class FeedController(AbstractController):
            icon_contr.create(**{"url": attrs["icon_url"]})

    def create(self, **attrs):
        assert (
            'link' in attrs
        ), "A feed must have a link."
        assert "link" in attrs, "A feed must have a link."
        self._ensure_icon(attrs)
        return super().create(**attrs)


M newspipe/crawler/default_crawler.py => newspipe/crawler/default_crawler.py +9 -5
@@ 112,8 112,7 @@ async def parse_feed(user, feed):


async def insert_articles(queue, nḅ_producers=1):
    """Consumer coroutines.
    """
    """Consumer coroutines."""
    nb_producers_done = 0
    while True:
        item = await queue.get()


@@ 172,14 171,19 @@ async def retrieve_feed(queue, users, feed_id=None):
        filters["last_retrieved__lt"] = datetime.now() - timedelta(
            minutes=application.config["FEED_REFRESH_INTERVAL"]
        )
        #feeds = FeedController().read(**filters).all()
        feeds = [] # temporary fix for: sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) SSL SYSCALL error: EOF detected
        # feeds = FeedController().read(**filters).all()
        feeds = (
            []
        )  # temporary fix for: sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) SSL SYSCALL error: EOF detected
        for feed in user.feeds:
            if not feed.enabled:
                continue
            if feed.error_count > application.config["DEFAULT_MAX_ERROR"]:
                continue
            if feed.last_retrieved > (datetime.now() - timedelta(minutes=application.config["FEED_REFRESH_INTERVAL"])):
            if feed.last_retrieved > (
                datetime.now()
                - timedelta(minutes=application.config["FEED_REFRESH_INTERVAL"])
            ):
                continue
            if None is feed_id or (feed_id and feed_id == feed.id):
                feeds.append(feed)

M newspipe/lib/article_utils.py => newspipe/lib/article_utils.py +5 -3
@@ 18,9 18,9 @@ PROCESSED_DATE_KEYS = {"published", "created", "updated"}


def extract_id(entry):
    """ extract a value from an entry that will identify it among the other of
    """extract a value from an entry that will identify it among the other of
    that feed"""
    entry_id = 'undefined'
    entry_id = "undefined"
    try:
        entry_id = entry.get("entry_id") or entry.get("id") or entry["link"]
    except:


@@ 49,7 49,9 @@ async def construct_article(entry, feed, fields=None, fetch=True):
                        timezone.utc
                    )
                except ParserError:
                    logger.exception("Error when parsing date: {}".format(entry[date_key]))
                    logger.exception(
                        "Error when parsing date: {}".format(entry[date_key])
                    )
                except Exception as e:
                    pass
                else:

M newspipe/lib/data.py => newspipe/lib/data.py +4 -6
@@ 112,7 112,7 @@ def import_json(nickname, json_content):
    nb_feeds, nb_articles = 0, 0
    # Create feeds:
    for feed in json_account:
        if 'link' not in feed.keys() or feed['link'] is None:
        if "link" not in feed.keys() or feed["link"] is None:
            continue
        if (
            None


@@ 134,7 134,7 @@ def import_json(nickname, json_content):
    db.session.commit()
    # Create articles:
    for feed in json_account:
        if 'link' not in feed.keys() or feed['link'] is None:
        if "link" not in feed.keys() or feed["link"] is None:
            continue
        user_feed = Feed.query.filter(
            Feed.user_id == user.id, Feed.link == feed["link"]


@@ 201,8 201,7 @@ def export_json(user):


def import_pinboard_json(user, json_content):
    """Import bookmarks from a pinboard JSON export.
    """
    """Import bookmarks from a pinboard JSON export."""
    bookmark_contr = BookmarkController(user.id)
    BookmarkTagController(user.id)
    bookmarks = json.loads(json_content.decode("utf-8"))


@@ 234,8 233,7 @@ def import_pinboard_json(user, json_content):


def export_bookmarks(user):
    """Export all bookmarks of a user (compatible with Pinboard).
    """
    """Export all bookmarks of a user (compatible with Pinboard)."""
    bookmark_contr = BookmarkController(user.id)
    bookmarks = bookmark_contr.read()
    export = []

M newspipe/lib/misc_utils.py => newspipe/lib/misc_utils.py +3 -2
@@ 101,11 101,12 @@ def fetch(id, feed_id=None):
    The default crawler ("asyncio") is launched with the manager.
    """
    env = os.environ.copy()
    env['FLASK_APP'] = 'runserver.py'
    env["FLASK_APP"] = "runserver.py"
    cmd = [
        sys.exec_prefix + "/bin/flask",
        "fetch_asyncio",
        "--user-id", str(id),
        "--user-id",
        str(id),
    ]
    if feed_id:
        cmd.extend(["--feed-id", str(feed_id)])

M newspipe/models/article.py => newspipe/models/article.py +9 -11
@@ 64,17 64,15 @@ class Article(db.Model, RightMixin):
    tags = association_proxy("tag_objs", "text")

    __table_args__ = (
            ForeignKeyConstraint([user_id], ['user.id'], ondelete='CASCADE'),
            ForeignKeyConstraint([feed_id], ['feed.id'], ondelete='CASCADE'),
            ForeignKeyConstraint([category_id], ['category.id'],
                                 ondelete='CASCADE'),
            Index('ix_article_eid_cid_uid', user_id, category_id, entry_id),
            Index('ix_article_retrdate', retrieved_date),

            Index('user_id'),
            Index('user_id', 'category_id'),
            Index('user_id', 'feed_id'),
            Index('ix_article_uid_fid_eid', user_id, feed_id, entry_id)
        ForeignKeyConstraint([user_id], ["user.id"], ondelete="CASCADE"),
        ForeignKeyConstraint([feed_id], ["feed.id"], ondelete="CASCADE"),
        ForeignKeyConstraint([category_id], ["category.id"], ondelete="CASCADE"),
        Index("ix_article_eid_cid_uid", user_id, category_id, entry_id),
        Index("ix_article_retrdate", retrieved_date),
        Index("user_id"),
        Index("user_id", "category_id"),
        Index("user_id", "feed_id"),
        Index("ix_article_uid_fid_eid", user_id, feed_id, entry_id),
    )

    # api whitelists

M newspipe/models/feed.py => newspipe/models/feed.py +5 -6
@@ 73,12 73,11 @@ class Feed(db.Model, RightMixin):
    )

    __table_args__ = (
            ForeignKeyConstraint([user_id], ['user.id'], ondelete='CASCADE'),
            ForeignKeyConstraint([category_id], ['category.id'],
                                 ondelete='CASCADE'),
            ForeignKeyConstraint([icon_url], ['icon.url']),
            Index('ix_feed_uid', user_id),
            Index('ix_feed_uid_cid', user_id, category_id),
        ForeignKeyConstraint([user_id], ["user.id"], ondelete="CASCADE"),
        ForeignKeyConstraint([category_id], ["category.id"], ondelete="CASCADE"),
        ForeignKeyConstraint([icon_url], ["icon.url"]),
        Index("ix_feed_uid", user_id),
        Index("ix_feed_uid_cid", user_id, category_id),
    )

    # idx_feed_uid_cid = Index("user_id", "category_id")

M newspipe/models/tag.py => newspipe/models/tag.py +2 -3
@@ 23,9 23,8 @@ class ArticleTag(db.Model):
        self.text = text

    __table_args__ = (
            ForeignKeyConstraint([article_id], ['article.id'],
                                 ondelete='CASCADE'),
            Index('ix_articletag_aid', article_id),
        ForeignKeyConstraint([article_id], ["article.id"], ondelete="CASCADE"),
        Index("ix_articletag_aid", article_id),
    )



M newspipe/notifications/notifications.py => newspipe/notifications/notifications.py +6 -2
@@ 46,7 46,9 @@ def new_account_notification(user, email):
    )

    emails.send(
        to=email, subject="[Newspipe] Account creation", plaintext=plaintext,
        to=email,
        subject="[Newspipe] Account creation",
        plaintext=plaintext,
    )




@@ 56,5 58,7 @@ def new_password_notification(user, password):
    """
    plaintext = render_template("emails/new_password.txt", user=user, password=password)
    emails.send(
        to=user.email, subject="[Newspipe] New password", plaintext=plaintext,
        to=user.email,
        subject="[Newspipe] New password",
        plaintext=plaintext,
    )

M newspipe/web/views/api/v2/common.py => newspipe/web/views/api/v2/common.py +1 -3
@@ 112,9 112,7 @@ class PyAggAbstractResource(Resource):

        if not default:
            for value in in_values:
                parser.add_argument(
                    value, location="json", default=in_values[value]
                )
                parser.add_argument(value, location="json", default=in_values[value])

        return parser.parse_args(req=request.args, strict=strict)


M newspipe/web/views/feed.py => newspipe/web/views/feed.py +3 -1
@@ 242,7 242,9 @@ def process_form(feed_id=None):

    if not form.validate():
        return render_template("edit_feed.html", form=form)
    existing_feeds = list(feed_contr.read(link=form.link.data, site_link=form.site_link.data))
    existing_feeds = list(
        feed_contr.read(link=form.link.data, site_link=form.site_link.data)
    )
    if existing_feeds and feed_id is None:
        flash(gettext("Couldn't add feed: feed already exists."), "warning")
        return redirect(url_for("feed.form", feed_id=existing_feeds[0].id))

M newspipe/web/views/home.py => newspipe/web/views/home.py +9 -13
@@ 21,8 21,7 @@ logger = logging.getLogger(__name__)
@current_app.route("/")
@login_required
def home():
    """Displays the home page of the connected user.
    """
    """Displays the home page of the connected user."""
    filters = _get_filters(request.args)

    category_contr = CategoryController(current_user.id)


@@ 76,17 75,14 @@ def home():
        search_title=search_title,
        search_content=search_content,
    ):
        return (
            "?filter_=%s&limit=%s&feed=%d&liked=%s&query=%s&search_title=%s&search_content=%s"
            % (
                filter_,
                limit,
                feed,
                1 if liked else 0,
                query,
                search_title,
                search_content,
            )
        return "?filter_=%s&limit=%s&feed=%d&liked=%s&query=%s&search_title=%s&search_content=%s" % (
            filter_,
            limit,
            feed,
            1 if liked else 0,
            query,
            search_title,
            search_content,
        )

    return render_template(

M newspipe/web/views/icon.py => newspipe/web/views/icon.py +6 -1
@@ 17,4 17,9 @@ def icon():
        return Response(base64.b64decode(icon.content), headers=headers)
    except:
        headers = {"Cache-Control": "max-age=86400", "Content-Type": "image/gif"}
        return Response(base64.b64decode("R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="), headers=headers)
        return Response(
            base64.b64decode(
                "R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="
            ),
            headers=headers,
        )

M newspipe/web/views/session_mgmt.py => newspipe/web/views/session_mgmt.py +3 -1
@@ 58,7 58,9 @@ def on_identity_loaded(sender, identity):
@login_manager.user_loader
def load_user(user_id):
    try:
        return UserController(user_id, ignore_context=True).get(id=user_id, is_active=True)
        return UserController(user_id, ignore_context=True).get(
            id=user_id, is_active=True
        )
    except NotFound:
        pass


M newspipe/web/views/views.py => newspipe/web/views/views.py +3 -1
@@ 107,5 107,7 @@ def about_more():
        ],
        python_version="{}.{}.{}".format(*sys.version_info[:3]),
        nb_users=UserController().read().count(),
        content_security_policy=talisman._parse_policy(talisman.content_security_policy),
        content_security_policy=talisman._parse_policy(
            talisman.content_security_policy
        ),
    )