~cedric/freshermeat

2c607bb0ed5808d29db82be089fd013f411e1700 — C├ędric Bonhomme 25 days ago 7749588
fetch_cve worker is now using cvepremium.
3 files changed, 89 insertions(+), 3 deletions(-)

M freshermeat/workers/fetch_cve.py
M poetry.lock
M pyproject.toml
M freshermeat/workers/fetch_cve.py => freshermeat/workers/fetch_cve.py +12 -2
@@ 23,6 23,7 @@ import asyncio
import logging
from datetime import datetime

import requests
from freshermeat.bootstrap import db
from freshermeat.models import CVE, get_or_create



@@ 33,9 34,18 @@ sem = asyncio.Semaphore(20)

async def get_cve(*args, **kwargs):
    try:
        data = # use the new CVE search API
        request_kwargs = {
            "verify": True,
            "allow_redirects": True,
            "timeout": 15,
            "headers": {"User-Agent": 'https://sr.ht/~cedric/freshermeat'},
        }
        result = requests.get("https://cvepremium.circl.lu/api/search/{}/{}".format(args[0], args[1]), **request_kwargs)
        logger.info("CVE for {} retrieved".format(args[2]))
        return data
        if result.status_code == 200:
            if result.json()['total'] != 0:
                return result.json()['results']
        return []
    except Exception as e:
        raise e


M poetry.lock => poetry.lock +76 -1
@@ 43,6 43,22 @@ optional = false
python-versions = "*"

[[package]]
name = "certifi"
version = "2020.12.5"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false
python-versions = "*"

[[package]]
name = "chardet"
version = "4.0.0"
description = "Universal encoding detector for Python 2 and 3"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"

[[package]]
name = "click"
version = "7.1.2"
description = "Composable command line interface toolkit"


@@ 264,6 280,14 @@ python-versions = ">=3.6"
tests = ["freezegun", "pytest", "pytest-cov"]

[[package]]
name = "idna"
version = "2.10"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"

[[package]]
name = "itsdangerous"
version = "1.1.0"
description = "Various helpers to pass data to untrusted environments and back."


@@ 465,6 489,24 @@ optional = false
python-versions = "*"

[[package]]
name = "requests"
version = "2.25.1"
description = "Python HTTP for Humans."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"

[package.dependencies]
certifi = ">=2017.4.17"
chardet = ">=3.0.2,<5"
idna = ">=2.5,<3"
urllib3 = ">=1.21.1,<1.27"

[package.extras]
security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]

[[package]]
name = "six"
version = "1.15.0"
description = "Python 2 and 3 compatibility utilities"


@@ 527,6 569,19 @@ python-versions = "*"
pytz = "*"

[[package]]
name = "urllib3"
version = "1.26.3"
description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"

[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]

[[package]]
name = "validate-email"
version = "1.3"
description = "Validate_email verify if an email address is valid and really exists."


@@ 565,7 620,7 @@ locale = ["Babel (>=1.3)"]
[metadata]
lock-version = "1.1"
python-versions = "^3.8"
content-hash = "b904bbed6982a86862f0159c7e2c459b9215a5f93b4d320af9dd2fa21727d97b"
content-hash = "bae0623f6c9e50e6cbc3c43f1e6a652059d1157c597c88455be20b8d84a9ddff"

[metadata.files]
alembic = [


@@ 582,6 637,14 @@ attrs = [
blinker = [
    {file = "blinker-1.4.tar.gz", hash = "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6"},
]
certifi = [
    {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"},
    {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"},
]
chardet = [
    {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"},
    {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"},
]
click = [
    {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
    {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},


@@ 643,6 706,10 @@ humanize = [
    {file = "humanize-3.2.0-py3-none-any.whl", hash = "sha256:d47d80cd47c1511ed3e49ca5f10c82ed940ea020b45b49ab106ed77fa8bb9d22"},
    {file = "humanize-3.2.0.tar.gz", hash = "sha256:ab69004895689951b79f2ae4fdd6b8127ff0c180aff107856d5d98119a33f026"},
]
idna = [
    {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
    {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
]
itsdangerous = [
    {file = "itsdangerous-1.1.0-py2.py3-none-any.whl", hash = "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"},
    {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"},


@@ 869,6 936,10 @@ regex = [
    {file = "regex-2020.11.13-cp39-cp39-win_amd64.whl", hash = "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d"},
    {file = "regex-2020.11.13.tar.gz", hash = "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562"},
]
requests = [
    {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"},
    {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"},
]
six = [
    {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
    {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},


@@ 924,6 995,10 @@ tzlocal = [
    {file = "tzlocal-2.1-py2.py3-none-any.whl", hash = "sha256:e2cb6c6b5b604af38597403e9852872d7f534962ae2954c7f35efcb1ccacf4a4"},
    {file = "tzlocal-2.1.tar.gz", hash = "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44"},
]
urllib3 = [
    {file = "urllib3-1.26.3-py2.py3-none-any.whl", hash = "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80"},
    {file = "urllib3-1.26.3.tar.gz", hash = "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"},
]
validate-email = [
    {file = "validate_email-1.3.tar.gz", hash = "sha256:784719dc5f780be319cdd185dc85dd93afebdb6ebb943811bc4c7c5f9c72aeaf"},
]

M pyproject.toml => pyproject.toml +1 -0
@@ 27,6 27,7 @@ feedparser = "^5.2.1"
psycopg2-binary = "^2.8.4"
flask_restx = "^0.2.0"
feedgen = "^0.9.0"
requests = "^2.25.1"


[tool.poetry.dev-dependencies]