~jb55/chibipub

8044145b1a5e7bd1d4b18c167f8c1b8081d39c50 — William Casarin a month ago 431f382
fetch signatures
3 files changed, 228 insertions(+), 45 deletions(-)

M Makefile
M default.nix
M src/sigcheck.c
M Makefile => Makefile +2 -2
@@ 1,6 1,6 @@

CFLAGS = -Og -ggdb -Wall -Werror -Ideps -Isrc $(shell pkg-config --cflags openssl)
LDFLAGS = $(shell pkg-config --libs openssl)
CFLAGS = -Og -ggdb -Wall -Werror -Ideps -Isrc $(shell pkg-config --cflags openssl libcurl)
LDFLAGS = $(shell pkg-config --libs openssl libcurl)

OBJS = src/http.o \
       src/base64.o \

M default.nix => default.nix +1 -1
@@ 3,5 3,5 @@ with pkgs;
stdenv.mkDerivation {
  name = "chibipub";
  nativeBuildInputs = [ gdb pkg-config ngrok clib ];
  buildInputs = [ openssl ];
  buildInputs = [ openssl curl ];
}

M src/sigcheck.c => src/sigcheck.c +225 -42
@@ 1,5 1,6 @@

#define SCRATCH_SIZE 1048576
#define SCRATCH_SIZE 134217728
#define MAX_PARALLEL 10

#include <sys/mman.h>
#include <sys/stat.h>


@@ 18,6 19,26 @@
#include "errors.h"

#include <ctype.h>
#include <curl/curl.h>

struct keyid_pubkey {
	unsigned char *data;
	int len;
};

enum key_writer_flags {
	KW_FAILED = 1 << 0,
	KW_STARTED = 1 << 1,
};

struct key_writer {
	CURL *curl;
	struct cursor *arena;
	struct keyid_pubkey *pubkey;
	const char *keyid;
	struct cursor slice;
	int flags;
};

static int verify_signature(struct cursor cur, struct cursor *arena)
{


@@ 36,30 57,26 @@ static int verify_signature(struct cursor cur, struct cursor *arena)
	return 1;
}

struct keyid {
	unsigned char *pubkey;
	int pubkey_size;
};

static inline int push_keyid(struct cursor *c, const char *keyid, int keyid_len)
{
	struct keyid offset = {0};
	struct keyid_pubkey pubkey = {0};
	return push_int(c, keyid_len)
		&& push_data(c, (unsigned char *)keyid, keyid_len)
		&& push_data(c, (unsigned char *)&offset, sizeof(offset)); // reserved for pubkey data index
		&& push_byte(c, 0)
		&& push_data(c, (unsigned char *)&pubkey, sizeof(pubkey)); // reserved for pubkey data index
}

static int pull_keyid(struct cursor *c, unsigned char **keyid_str,
		int *keyid_len, struct keyid **keyid)
static int pull_keyid(struct cursor *c, const char **keyid_str, int *keyid_len,
		struct keyid_pubkey **pubkey)
{
	if (!pull_int(c, keyid_len)) {
		return 0;
	}

	*keyid_str = c->p;
	c->p += *keyid_len;
	*keyid = (struct keyid*)c->p;
	c->p += sizeof(struct keyid);
	*keyid_str = (const char*)c->p;
	c->p += (*keyid_len) + 1; /* plus zero */
	*pubkey = (struct keyid_pubkey*)c->p;
	c->p += sizeof(struct keyid_pubkey);

	return 1;
}


@@ 68,8 85,8 @@ static int has_keyid(struct cursor *keyids, const char *keyid, int keyid_len)
{
	struct cursor scan;
	int size;
	unsigned char *keyid_str;
	struct keyid *offset;
	const char *keyid_str;
	struct keyid_pubkey *offset;

	scan.start = keyids->start;
	scan.p = keyids->start;


@@ 90,20 107,18 @@ static int has_keyid(struct cursor *keyids, const char *keyid, int keyid_len)
	return 0;
}

static int print_keyids(struct cursor *keyids)
static int print_keyids(struct cursor keyids)
{
	struct cursor scan;
	int size;
	unsigned char *keyid_str;
	struct keyid *keyid;
	const char *keyid_str;
	struct keyid_pubkey *pubkey;

	scan.start = keyids->start;
	scan.p = keyids->start;
	scan.end = keyids->p;
	keyids.end = keyids.p;
	keyids.p = keyids.start;

	printf("keyids\n");
	while (scan.p < scan.end) {
		if (!pull_keyid(&scan, &keyid_str, &size, &keyid))
	while (keyids.p < keyids.end) {
		if (!pull_keyid(&keyids, &keyid_str, &size, &pubkey))
			return 0;
		printf("%.*s\n", size, keyid_str);
	}


@@ 148,7 163,11 @@ static int gather_keyids(unsigned char *json, int json_len,
		}
	}

	print_keyids(keyids_cur);
	keyids_cur->end = keyids_cur->p;
	keyids_cur->p = keyids_cur->start;
	arena->p = keyids_cur->end;

	print_keyids(*keyids_cur);

	return 1;
}


@@ 184,7 203,7 @@ static int get_cached_pubkey(unsigned char *keyid, int size,
	if (0 == mkdir(".chibipub", mode)) {
		return 0;
	}
	

	if (0 == mkdir(".chibipub/objects", mode)) {
		return 0;
	}


@@ 197,7 216,7 @@ static int get_cached_pubkey(unsigned char *keyid, int size,
	sprintf(path, ".chibipub/objects/%c%c/%.*s",
			hash_str[0], hash_str[1], 64, hash_str);

	fprintf(stderr, "sig cache exists? '%s' ", path);
	fprintf(stderr, "key cache exists? '%s' ", path);
	if (access(path, F_OK)) {
		fprintf(stderr, "no.\n");
		return 0;


@@ 207,9 226,144 @@ static int get_cached_pubkey(unsigned char *keyid, int size,
	return read_file(path, buf, buflen, pubkey_size);
}

static int fetch_signature(unsigned char *keyid, int keyid_len)
static size_t write_cb(char *data, size_t n, size_t l, void *userp)
{
	double dcl;
	int cl;
	int res;
	size_t size = n*l;
	struct key_writer *writer = (struct key_writer *)userp;

	if (!(writer->flags & KW_STARTED)) {
		res = curl_easy_getinfo(writer->curl,
				CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dcl);
		if (res) {
			fprintf(stderr, "'%s' download failed, content-length unknown\n",
					writer->keyid);
			writer->flags |= KW_FAILED;
			return 0;
		}

		cl = (int)dcl;
		if (cl == -1) {
			cl = 1024*1024;
		}

		writer->pubkey->data = cursor_alloc(writer->arena, cl);
		if (!writer->pubkey->data) {
			fprintf(stderr, "cursor_alloc failed, remaining (%ld) + %d (%ld total)\n",
				writer->arena->end - writer->arena->p,
				cl, writer->arena->end - writer->arena->start);
			writer->flags |= KW_FAILED;
			return 0;
		}

		writer->flags |= KW_STARTED;
		writer->pubkey->len = 0;
		make_cursor(writer->pubkey->data, writer->pubkey->data + cl,
				&writer->slice);
	}

	if (!push_data(&writer->slice, (unsigned char*)data, size)) {
		writer->flags |= KW_FAILED;
		return 0;
	}

	writer->pubkey->len += size;

	return size;
}

static int add_signature_transfer(CURLM *cm, struct key_writer *writer)
{
	printf("fetch signature %.*s\n", keyid_len, keyid);
	CURL *eh = curl_easy_init();
	writer->curl = eh;
	curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, write_cb);
	curl_easy_setopt(eh, CURLOPT_WRITEDATA, writer);
	curl_easy_setopt(eh, CURLOPT_URL, writer->keyid);
	curl_easy_setopt(eh, CURLOPT_PRIVATE, writer);
	curl_multi_add_handle(cm, eh);

	return 1;
}

static void prepare_transfers(CURLM **cm)
{
	curl_global_init(CURL_GLOBAL_ALL);
	*cm = curl_multi_init();
	curl_multi_setopt(*cm, CURLMOPT_MAXCONNECTS, (long)10);
}

static int handle_transfer_msg(CURLM *cm, CURLMsg *msg, int *transfers,
		struct key_writer *to_fetch, int n_to_fetch)
{
	struct key_writer *writer;

	if (msg->msg == CURLMSG_DONE) {
		CURL *e = msg->easy_handle;

		curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &writer);

		fprintf(stderr, "R: %d - %s <%s>\n",
				msg->data.result,
				curl_easy_strerror(msg->data.result),
				writer->keyid);

		curl_multi_remove_handle(cm, e);
		curl_easy_cleanup(e);
	} else {
		fprintf(stderr, "E: CURLMsg (%d)\n", msg->msg);
	}

	if (*transfers < n_to_fetch) {
		add_signature_transfer(cm, &to_fetch[*transfers]);
		*transfers = *transfers + 1;
	}

	return 1;
}

static int perform_transfers(CURLM *cm, int transfers,
		struct key_writer *to_fetch, int n_to_fetch)
{
	CURLMsg *msg;
	int still_alive = 1;
	int msgs_left = -1;

	do {
		curl_multi_perform(cm, &still_alive);

		while ((msg = curl_multi_info_read(cm, &msgs_left))) {
			handle_transfer_msg(
				cm, msg, &transfers, to_fetch, n_to_fetch);
		}

		if (still_alive) {
			curl_multi_wait(cm, NULL, 0, 1000, NULL);
		}
	} while (still_alive || (transfers < n_to_fetch));

	curl_multi_cleanup(cm);
	curl_global_cleanup();

	return 1;
}

static int count_keyids(struct cursor scan, int *count)
{
	int size;
	const char *keyid_str;
	struct keyid_pubkey *pubkey;

	*count = 0;

	while (scan.p < scan.end) {
		if (!pull_keyid(&scan, &keyid_str, &size, &pubkey)) {
			return 0;
		}
		*count = *count + 1;
	}

	return 1;
}



@@ 218,34 372,63 @@ static int fetch_signatures(unsigned char *json, int json_len,
{
	struct cursor keyids;
	struct cursor scan;
	int size;
	unsigned char *keyid_str;
	struct keyid *keyid;
	int size, transfers;
	int n_to_fetch, n_keyids;
	const char *keyid_str;
	struct key_writer *to_fetch, *kw;
	struct keyid_pubkey *pubkey;
	CURLM *cm;
	n_to_fetch = 0;

	if (!gather_keyids(json, json_len, arena, &keyids)) {
		return 0;
	}

	scan.start = keyids.start;
	scan.p = scan.start;
	scan.end = keyids.p;
	prepare_transfers(&cm);

	scan.p = scan.start = keyids.start;
	scan.end = keyids.end;

	if (!count_keyids(scan, &n_keyids)) {
		return 0;
	}

	if (!(to_fetch = cursor_alloc(arena, n_keyids * sizeof(struct key_writer)))) {
		return 0;
	}

	while (scan.p < scan.end) {
		if (!pull_keyid(&scan, &keyid_str, &size, &keyid)) {

		if (!pull_keyid(&scan, &keyid_str, &size, &pubkey)) {
			return 0;
		}

		if (get_cached_pubkey(scan.p, size, arena->p,
				      arena->end - arena->p,
				      &keyid->pubkey_size)) {
			keyid->pubkey = arena->p;
			arena->p += keyid->pubkey_size;
				      &pubkey->len)) {
			pubkey->data = arena->p;
			arena->p += pubkey->len;
			printf("got cached pubkey of size %d... do something\n",
					keyid->pubkey_size);
					pubkey->len);

			continue;
		}

		fetch_signature(keyid_str, size);
		pubkey->len = 0;

		kw = &to_fetch[n_to_fetch++];
		kw->keyid = keyid_str;
		kw->arena = arena;
		kw->pubkey = pubkey;
		kw->flags = 0;
	}

	for (transfers = 0; transfers < MAX_PARALLEL && transfers < n_to_fetch; transfers++) {
		add_signature_transfer(cm, &to_fetch[transfers]);
	}

	if (!perform_transfers(cm, transfers, to_fetch, n_to_fetch)) {
		return 0;
	}

	return 1;