~prokop/lua-pebble

ccead104d0d8736b565376c7173233564c505d88 — Prokop Randacek 1 year, 7 months ago 90a8626 
targz support
patch browse .tar.gz 
4 files changed, 135 insertions(+), 42 deletions(-)

R example.lua => example_github.lua
A example_github_tag.lua
R example2.lua => example_srht_paste.lua
M pebble.lua

R example.lua => example_github.lua +0 -0
A example_github_tag.lua => example_github_tag.lua +8 -0
@@ 0,0 1,8 @@
-- SPDX-License-Identifier: MIT
require('pebble') -- init pebble

-- load a file from github using its pebble string
local fun = require('ghtag!luafun/luafun@0.1.3!fun.lua')

-- use the library as usual
print(fun.range(100):map(function(x) return x ^ 2 end):reduce(fun.op.add, 0))


R example2.lua => example_srht_paste.lua +0 -0
M pebble.lua => pebble.lua +127 -42
@@ 1,51 1,75 @@
-- SPDX-License-Identifier: MIT
-- TODO: archive extracting fetcher
-- https://git.sr.ht/~prokop/lua-pebble
local cache = os.getenv('HOME') .. '/.cache/luapebble'

-- luajit compat
-- lua5.1 compat
---@diagnostic disable-next-line: deprecated
local unpack, searchers = table.unpack or unpack, package.searchers or package.loaders

local function system(...) -- executes cmd and returns true if retcode is zero
	local r, s, n = os.execute(string.format(...))
local conf = _PEBBLE or {}
local week = (60 * 60 * 24 * 7)
local refetch_time = conf.max_age_seconds or week

local function say(msg) if not conf.quiet then io.write('pebble: ' .. msg .. '\n') end end
local function verbose(msg) if conf.verbose then io.write('pebble: ' .. msg .. '\n') end end

local function run(cmd) -- runs cmd and returns true if retcode is zero plus the command to be used as assert failure message
	verbose(cmd)
	local r, s, n = os.execute(cmd)
	if s then
		return s == 'exit' and n == 0 -- 5.4
		return s == 'exit' and n == 0, cmd -- 5.4
	else
		return r == 0   -- 5.1
		return r == 0, cmd   -- 5.1
	end
end

local function system2(...) -- executes cmd and returns its stdout
	return io.popen(string.format(...)):read("*a")
local function run_stdout(cmd) -- executes cmd and returns its stdout
	verbose("sh: " .. cmd)
	return io.popen(cmd):read("*a")
end

local function dir_exists(d) return system('[ -d %s ]', d) end
local function file_exists(d) return system('[ -f %s ]', d) end
local function mkdir(dir) assert(run('mkdir -p -- "' .. dir .. '"')) end
local function mkfile(file) assert(run('touch -- "' .. file .. '"')) end
local function rmdir(dir) assert(run('rm -rf -- "' .. dir .. '"')) end
local function rmfile(file) assert(run('rm -f -- "' .. file .. '"')) end
local function dir_missing(d) return not run('[ -d "' .. d .. '" ]') end
local function file_missing(d) return not run('[ -f "' .. d .. '" ]') end
local function file_older_than(d, t)
	local file_time = assert(tonumber(system2('stat -c "%%Y" "%s"', d)))
	local file_time = assert(tonumber(run_stdout('stat -c "%Y" "' .. d .. '"')))
	return os.difftime(os.time(), file_time) > t
end

local function git_clone(dir, url) return run('git clone --quiet --recurse -- "' .. url .. '" "' .. dir .. '"') end
local function git_pull(dir)
	return run('git pull --quiet -- "' .. dir .. '"') and
	    run('git submodule update --init --recursive -- "' .. dir .. '"')
end
local function curl(url, file) return run('curl --silent --location "' .. url .. '" -o "' .. file .. '"') end
local function lstar(file, dir) return run_stdout('tar -tf "' .. file .. '"') end
local function untar(file, dir, s)
	return run('tar -xzf "' ..
		file .. '" --directory "' .. dir .. '" --strip-components=' .. (s or 0))
end

local fetchers = {}

function fetchers.git(url, file, safe_name, max_age)
	local target_dir = cache .. '/' .. safe_name
function fetchers.git(url, file, name, max_age)
	local target_dir = cache .. '/' .. name
	local target_file = target_dir .. '/' .. file
	local stamp_file = target_dir .. '.timestamp'
	if not dir_exists(target_dir) then
		assert(system('mkdir -p -- "%s"', target_dir))
		assert(system('touch -- "%s"', stamp_file))
		io.write(string.format('pebble: fetching %s\n', url))
		if not system('git clone --quiet --recurse-submodules --shallow-submodules -- "%s" "%s"', url, target_dir) then
			assert(system('rm -rf -- "%s"', target_dir))
	if dir_missing(target_dir) then
		mkdir(target_dir)
		mkfile(stamp_file)
		say('fetching ' .. url)
		if not git_clone(target_dir, url) then
			rmdir(target_dir)
			return 'pebble: git clone failed'
		end
	elseif file_older_than(stamp_file, max_age) then
		io.write(string.format('pebble: updating %s\n', url))
		if not system('git pull --quiet -- "%s"', target_dir) then
		say('updating ' .. url)
		if not git_pull(target_dir) then
			return 'pebble: git pull failed'
		end
		assert(system('touch -- "%s"', stamp_file))
		mkfile(stamp_file)
	end
	local f, err = loadfile(target_file, 't')
	if not f then return err end


@@ 58,12 82,64 @@ function fetchers.git(url, file, safe_name, max_age)
	end
end

function fetchers.file(url, _, safe_name, max_age)
function fetchers.targz(url, file, name, max_age)
	local target_dir = cache .. '/' .. name
	local target_file = target_dir .. '/' .. file
	local stamp_file = target_dir .. '.timestamp'
	local tar_file = cache .. '/' .. name .. '.tar.gz'
	local fetch = false
	if dir_missing(target_dir) then
		mkdir(target_dir)
		fetch = true
	elseif file_older_than(stamp_file, max_age) then
		rmdir(target_dir)
		fetch = true
	end
	if fetch then
		mkfile(stamp_file)
		say('fetching ' .. url)
		if not curl(url, tar_file) then
			rmdir(target_dir)
			return 'pebble: curl failed'
		end

		-- check if all files are inside single directory
		-- and if so, strip the first dir.
		local files = lstar(tar_file)
		local skip_dirs = 1
		local prefix = string.match(files, "^[^/]+")
		for line in string.gmatch(files, "[^\n]+") do
			if string.match(line, "^[^/]+") ~= prefix then
				skip_dirs = 0
				break
			end
		end
		if not untar(tar_file, target_dir, skip_dirs) then
			rmdir(target_dir)
			return 'pebble: tar xzf failed'
		end
		--rmfile(tar_file)
	end
	local f, err = loadfile(target_file, 't')
	if not f then return err end
	return function()
		local oldpath = package.path
		package.path = target_dir .. '/?.lua;' .. package.path
		local r = { f(target_file) }
		package.path = oldpath
		return unpack(r)
	end
end

function fetchers.file(url, file, safe_name, max_age)
	if file then say("ignoring unwanted file specifier for file fetcher: '" .. file .. "'") end
	local target_file = cache .. '/' .. safe_name
	if not file_exists(target_file) or file_older_than(target_file, max_age) then
		system('mkdir -p -- %s', cache)
		io.write(string.format('pebble: fetching %s\n', url))
		system('curl --silent "%s" -o "%s"', url, target_file)
	if file_missing(target_file) or file_older_than(target_file, max_age) then
		mkdir(cache)
		say('fetching ' .. url)
		if not curl(url, target_file) then
			return "pebble: curl failed"
		end
	end
	local f, err = loadfile(target_file, 't')
	if not f then return err end


@@ 72,29 148,38 @@ function fetchers.file(url, _, safe_name, max_age)
	end
end

local week = (60 * 60 * 24 * 7)

local aliases = { -- url format string, fetching function, max age before refetch
	git = { '%s', fetchers.git, week },
	srht = { 'https://git.sr.ht/%s', fetchers.git, week },
	gh = { 'https://github.com/%s.git', fetchers.git, week },
	gist = { 'https://gist.github.com/%s.git', fetchers.git, math.huge },
	file = { '%s', fetchers.file, week },
	pastebin = { 'https://pastebin.com/raw/%s', fetchers.file, math.huge },
local aliases = {
	-- url format string, fetching function, max age before refetch
	git       = { '%s', fetchers.git, refetch_time },
	srht      = { 'https://git.sr.ht/%s', fetchers.git, refetch_time },
	gh        = { 'https://github.com/%s.git', fetchers.git, refetch_time },
	gist      = { 'https://gist.github.com/%s.git', fetchers.git, math.huge },
	file      = { '%s', fetchers.file, refetch_time },
	pastebin  = { 'https://pastebin.com/raw/%s', fetchers.file, math.huge },
	pastesrht = { 'https://paste.sr.ht/blob/%s', fetchers.file, math.huge },
	targz     = { '%s', fetchers.targz, math.huge },
	ghtag     = { 'https://github.com/%s/archive/refs/tags/%s.tar.gz', fetchers.targz, math.huge },
}

table.insert(searchers, function(modname)
	local t = {}
	for w in string.gmatch(modname, "[^%G!]+") do table.insert(t, w) end
	for w in string.gmatch(modname, "[^%G!]+") do
		table.insert(t, w)
	end

	local kind, name, file = unpack(t)
	local kind, name, file, e = unpack(t)

	if not kind or not name then return "pebble: string didn't match the pattern" end
	if not aliases[kind] then return string.format("pebble: unknown service '%s'", kind) end
	if e then return "pebble: string has too many '!'" end
	if not kind or not name then return "pebble: string doesn't have enough '!'" end
	if not aliases[kind] then return "pebble: unknown service '" .. kind .. "'" end

	local t_name = {}
	for w in string.gmatch(name, '[^@]+') do
		table.insert(t_name, w)
	end

	local urlfmt, fetcher, max_age = unpack(aliases[kind])
	local url = string.format(urlfmt, name)
	local url = string.format(urlfmt, unpack(t_name))
	local safename = kind .. '_' .. string.gsub(url, "%W", "_")

	return fetcher(url, file, safename, max_age)