~cadence/bibliogram

271afec9db276fe073e82ae306014ab9c38c3e35 — Cadence Ember a month ago db36472
Add VPN list and applied-privacy subnet list
M package-lock.json => package-lock.json +5 -0
@@ 2175,6 2175,11 @@
        "is-extglob": "^2.1.1"
      }
    },
    "is-in-subnet": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/is-in-subnet/-/is-in-subnet-3.1.0.tgz",
      "integrity": "sha512-UpHSoZ+S53dhiM5q48atdahm6GmaTFlZaelKTdtBQeSihWZdToE7b8hlTzWVVLghntGlCK97c1JTIdzV5Q+Vcw=="
    },
    "is-number": {
      "version": "7.0.0",
      "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",

M package.json => package.json +1 -0
@@ 20,6 20,7 @@
    "cookie": "^0.4.1",
    "feed": "git+https://git.sr.ht/~cadence/nodejs-feed#3dde82f8296d7a6f5659323e497e0c684f03ab71",
    "get-stream": "^5.1.0",
    "is-in-subnet": "^3.1.0",
    "mixin-deep": "^2.0.1",
    "node-dir": "^0.1.17",
    "node-fetch": "^2.6.0",

M scripts/process_log.js => scripts/process_log.js +1 -1
@@ 175,7 175,7 @@ reader.on("line", line => {
	if (kind) {
		kinds[kind]++
		dateCollection.add(kind, dateObject.getTime())
		if (kind === "api") ips.add(parsed.ip)
		ips.add(parsed.ip)
	}
})


M src/lib/quota/get_identifier.js => src/lib/quota/get_identifier.js +41 -13
@@ 1,25 1,53 @@
const {request} = require("../utils/request")
const {log} = require("pinski/util/common")
const constants = require("../constants.js")
const {createChecker} = require("is-in-subnet")

let addresses = []
let addressSet = new Set()
let subnets = []
let checker = createChecker([])

request("https://check.torproject.org/torbulkexitlist").text().then(text => {
	const lines = text.split("\n").filter(l => l)
	addresses = addresses.concat(lines)
	log(`Loaded Tor exit node list (${addresses.length} total)`, "spam")
})
function getList(url, description) {
	return request(url).text().then(text => {
		// this is decently fast, but if you have ideas for optimisation, please go for it
		let d = Date.now()
		const lines = text.split("\n").filter(l => l && l[0] !== "#")
		subnets = subnets.concat(lines.filter(l => l.includes("/")))
		addresses = addresses.concat(lines.filter(l => !l.includes("/")))
		log(`Loaded ${description} (entries: ${lines.length}) (${Date.now()-d} ms)`, "spam")
	})
}

/*
	request("https://meta.bibliogram.art/ip_proxy_list.txt").text().then(text => {
		const lines = text.split("\n").filter(l => l)
		addresses = addresses.concat(lines)
		log(`Loaded Bibliogram proxy list (${addresses.length} total)`, "spam")
if (constants.quota.enabled) {
	Promise.all([
		getList("https://check.torproject.org/torbulkexitlist", "Tor exit node list"),
		getList("https://meta.bibliogram.art/quota-list/vpn-ipv4.txt", "VPN IPv4 list"),
		getList("https://meta.bibliogram.art/quota-list/applied-privacy.txt", "applied-privacy.net subnets")
	]).then(() => {
		let d = Date.now()
		checker = createChecker(subnets)
		addressSet = new Set(addresses.values())
		log(`Created subnet checker (${Date.now()-d} ms)`, "spam")
	})
*/
}

function getIdentifier(address) {
	if (addresses.includes(address)) return "proxy"
	else return address
	let d = Date.now()
	const result = (() => {
		try {
			if (address == undefined) return "missing"
			else if (checker(address)) return "proxy"
			else if (addressSet.has(address)) return "proxy"
			else return address
		} catch (e) {
			// not a valid IP address, or some error like that
			console.error(e)
			throw e
		}
	})()
	console.log(`identified ${address} -> ${result} in ${Date.now()-d} ms`)
	return result
}

module.exports.getIdentifier = getIdentifier

M src/lib/quota/index.js => src/lib/quota/index.js +2 -2
@@ 20,7 20,7 @@ function remaining(req) {
	if (!constants.quota.enabled) return Infinity // sure.

	const ip = getIPFromReq(req)
	const identifier = String(getIdentifier(ip))
	const identifier = getIdentifier(ip)
	const remaining = limiter.remaining(identifier)

	if (constants.quota.track) {


@@ 34,7 34,7 @@ function add(req, count) {
	if (!constants.quota.enabled) return Infinity // why not.

	const ip = getIPFromReq(req)
	const identifier = String(getIdentifier(ip))
	const identifier = getIdentifier(ip)
	return limiter.add(identifier, count)
}