~kennylevinsen/seatd

b3438f446e19f9cbe37724a1efe070d03baa292b — Kenny Levinsen 8 months ago 7252558 connection_experiment
connection: Shrink buffer sizes

A connection contains 4 connection buffers: in, out, fds_in and fds_out.
The connection_buffer struct previously embedded a fixed-size 1KB
ring-buffer, leading to the connection structure being in excess of 4KB.

This is way more buffer than is needed, especially for fd buffers where
only 8 fds were allowed buffered, and only one direction was ever
needed.  However, as the buffer is inline in the generic
connection_buffer struct, the sizes cannot differ between types.

To allow per-connection_buffer sizing, the connection_buffer struct is
changed to have a char pointer and a capacity instead of an inline
buffer. Then, to avoid needing to alloc the buffers, a shared buffer is
placed in the connection struct. To set this all up, an init method is
added to the connection. This allows the fd buffers to be sizeof(int)*8,
while data buffers can be 256 bytes.

With this, raw buffer size for a connection drops from 4KB to 544B.
M common/connection.c => common/connection.c +45 -32
@@ 1,3 1,4 @@
#include <assert.h>
#include <errno.h>
#include <stddef.h>
#include <stdio.h>


@@ 6,24 7,22 @@
#include <sys/socket.h>
#include <unistd.h>

#include "compiler.h"
#include "connection.h"

#define CLEN (CMSG_LEN(MAX_FDS_OUT * sizeof(int)))

ALWAYS_INLINE static uint32_t connection_buffer_mask(const uint32_t idx) {
	return idx & (CONNECTION_BUFFER_SIZE - 1);
static inline uint16_t connection_buffer_mask(const struct connection_buffer *b, const uint16_t idx) {
	assert((b->capacity & (b->capacity - 1)) == 0);
	return idx & (b->capacity - 1);
}

ALWAYS_INLINE static uint32_t connection_buffer_size(const struct connection_buffer *b) {
static inline uint16_t connection_buffer_size(const struct connection_buffer *b) {
	return b->head - b->tail;
}

ALWAYS_INLINE static void connection_buffer_consume(struct connection_buffer *b, const size_t size) {
static inline void connection_buffer_consume(struct connection_buffer *b, const size_t size) {
	b->tail += size;
}

ALWAYS_INLINE static void connection_buffer_restore(struct connection_buffer *b, const size_t size) {
static inline void connection_buffer_restore(struct connection_buffer *b, const size_t size) {
	b->tail -= size;
}



@@ 32,19 31,19 @@ ALWAYS_INLINE static void connection_buffer_restore(struct connection_buffer *b,
 * Two may be used if the buffer has wrapped around.
 */
static void connection_buffer_get_iov(struct connection_buffer *b, struct iovec *iov, int *count) {
	uint32_t head = connection_buffer_mask(b->head);
	uint32_t tail = connection_buffer_mask(b->tail);
	uint16_t head = connection_buffer_mask(b, b->head);
	uint16_t tail = connection_buffer_mask(b, b->tail);
	if (tail < head) {
		iov[0].iov_base = b->data + tail;
		iov[0].iov_len = head - tail;
		*count = 1;
	} else if (head == 0) {
		iov[0].iov_base = b->data + tail;
		iov[0].iov_len = sizeof b->data - tail;
		iov[0].iov_len = b->capacity - tail;
		*count = 1;
	} else {
		iov[0].iov_base = b->data + tail;
		iov[0].iov_len = sizeof b->data - tail;
		iov[0].iov_len = b->capacity - tail;
		iov[1].iov_base = b->data;
		iov[1].iov_len = head;
		*count = 2;


@@ 56,19 55,19 @@ static void connection_buffer_get_iov(struct connection_buffer *b, struct iovec 
 * Two may be used if the buffer has wrapped around.
 */
static void connection_buffer_put_iov(struct connection_buffer *b, struct iovec *iov, int *count) {
	uint32_t head = connection_buffer_mask(b->head);
	uint32_t tail = connection_buffer_mask(b->tail);
	uint16_t head = connection_buffer_mask(b, b->head);
	uint16_t tail = connection_buffer_mask(b, b->tail);
	if (head < tail) {
		iov[0].iov_base = b->data + head;
		iov[0].iov_len = tail - head;
		*count = 1;
	} else if (tail == 0) {
		iov[0].iov_base = b->data + head;
		iov[0].iov_len = sizeof b->data - head;
		iov[0].iov_len = b->capacity - head;
		*count = 1;
	} else {
		iov[0].iov_base = b->data + head;
		iov[0].iov_len = sizeof b->data - head;
		iov[0].iov_len = b->capacity - head;
		iov[1].iov_base = b->data;
		iov[1].iov_len = tail;
		*count = 2;


@@ 79,13 78,13 @@ static void connection_buffer_put_iov(struct connection_buffer *b, struct iovec 
 * connection_buffer_copy copies from our ring buffer into a linear buffer.
 */
static void connection_buffer_copy(const struct connection_buffer *b, void *data, const size_t count) {
	uint32_t tail = connection_buffer_mask(b->tail);
	if (tail + count <= sizeof b->data) {
	uint16_t tail = connection_buffer_mask(b, b->tail);
	if (tail + count <= b->capacity) {
		memcpy(data, b->data + tail, count);
		return;
	}

	uint32_t size = sizeof b->data - tail;
	uint16_t size = b->capacity - tail;
	memcpy(data, b->data + tail, size);
	memcpy((char *)data + size, b->data, count - size);
}


@@ 93,16 92,16 @@ static void connection_buffer_copy(const struct connection_buffer *b, void *data
 * connection_buffer_copy copies from a linear buffer into our ring buffer.
 */
static int connection_buffer_put(struct connection_buffer *b, const void *data, const size_t count) {
	if (count > sizeof(b->data)) {
	if (count > b->capacity) {
		errno = EOVERFLOW;
		return -1;
	}

	uint32_t head = connection_buffer_mask(b->head);
	if (head + count <= sizeof b->data) {
	uint16_t head = connection_buffer_mask(b, b->head);
	if (head + count <= b->capacity) {
		memcpy(b->data + head, data, count);
	} else {
		uint32_t size = sizeof b->data - head;
		uint16_t size = b->capacity - head;
		memcpy(b->data + head, data, size);
		memcpy(b->data, (const char *)data + size, count - size);
	}


@@ 119,7 118,8 @@ static void connection_buffer_close_fds(struct connection_buffer *buffer) {
	if (size == 0) {
		return;
	}
	int fds[sizeof(buffer->data) / sizeof(int)];
	assert(buffer->capacity / sizeof(int) == MAX_FDS);
	int fds[MAX_FDS];
	connection_buffer_copy(buffer, fds, size);
	int count = size / sizeof fds[0];
	size = count * sizeof fds[0];


@@ 134,8 134,8 @@ static void connection_buffer_close_fds(struct connection_buffer *buffer) {
 */
static void build_cmsg(struct connection_buffer *buffer, char *data, int *clen) {
	size_t size = connection_buffer_size(buffer);
	if (size > MAX_FDS_OUT * sizeof(int)) {
		size = MAX_FDS_OUT * sizeof(int);
	if (size > buffer->capacity) {
		size = buffer->capacity;
	}

	if (size <= 0) {


@@ 160,7 160,7 @@ static int decode_cmsg(struct connection_buffer *buffer, struct msghdr *msg) {
		}

		size_t size = cmsg->cmsg_len - CMSG_LEN(0);
		size_t max = sizeof(buffer->data) - connection_buffer_size(buffer);
		size_t max = buffer->capacity - connection_buffer_size(buffer);
		if (size > max || overflow) {
			overflow = true;
			size /= sizeof(int);


@@ 179,8 179,21 @@ static int decode_cmsg(struct connection_buffer *buffer, struct msghdr *msg) {
	return 0;
}

void connection_init(struct connection *connection) {
	connection->in.capacity = CONNECTION_BUFFER_SIZE;
	connection->out.capacity = CONNECTION_BUFFER_SIZE;
	connection->fds_in.capacity = CONNECTION_FD_BUFFER_SIZE;
	connection->fds_out.capacity = CONNECTION_FD_BUFFER_SIZE;

	connection->in.data = connection->connection_data;
	connection->out.data = connection->in.data + CONNECTION_BUFFER_SIZE;
	connection->fds_in.data = connection->out.data + CONNECTION_FD_BUFFER_SIZE;
	connection->fds_out.data = connection->fds_in.data + CONNECTION_FD_BUFFER_SIZE;
	return;
}

int connection_read(struct connection *connection) {
	if (connection_buffer_size(&connection->in) >= sizeof(connection->in.data)) {
	if (connection_buffer_size(&connection->in) >= connection->in.capacity) {
		errno = EOVERFLOW;
		return -1;
	}


@@ 189,7 202,7 @@ int connection_read(struct connection *connection) {
	struct iovec iov[2];
	connection_buffer_put_iov(&connection->in, iov, &count);

	char cmsg[CLEN];
	char cmsg[CMSG_LEN(CONNECTION_FD_BUFFER_SIZE)];
	struct msghdr msg = {
		.msg_name = NULL,
		.msg_namelen = 0,


@@ 220,14 233,14 @@ int connection_flush(struct connection *connection) {
		return 0;
	}

	uint32_t tail = connection->out.tail;
	uint16_t tail = connection->out.tail;
	while (connection->out.head - connection->out.tail > 0) {
		int count;
		struct iovec iov[2];
		connection_buffer_get_iov(&connection->out, iov, &count);

		int clen;
		char cmsg[CLEN];
		char cmsg[CMSG_LEN(CONNECTION_FD_BUFFER_SIZE)];
		build_cmsg(&connection->fds_out, cmsg, &clen);
		struct msghdr msg = {
			.msg_name = NULL,


@@ 269,7 282,7 @@ int connection_put(struct connection *connection, const void *data, size_t count
}

int connection_put_fd(struct connection *connection, int fd) {
	if (connection_buffer_size(&connection->fds_out) == MAX_FDS_OUT * sizeof fd) {
	if (connection_buffer_size(&connection->fds_out) >= connection->fds_out.capacity) {
		errno = EOVERFLOW;
		return -1;
	}

M include/compiler.h => include/compiler.h +0 -2
@@ 9,10 9,8 @@

#if defined(__GNUC__) && __GNUC__ >= 4
#define LIBSEAT_EXPORT __attribute__((visibility("default")))
#define ALWAYS_INLINE  __attribute__((always_inline)) inline
#else
#define LIBSEAT_EXPORT
#define ALWAYS_INLINE inline
#endif

#define STRLEN(s) ((sizeof(s) / sizeof(s[0])) - 1)

M include/connection.h => include/connection.h +9 -5
@@ 4,22 4,26 @@
#include <stddef.h>
#include <stdint.h>

#define CONNECTION_BUFFER_SIZE 1024

#define MAX_FDS_OUT 8
#define MAX_FDS			  4
#define CONNECTION_BUFFER_SIZE	  256
#define CONNECTION_FD_BUFFER_SIZE (MAX_FDS * sizeof(int))

struct connection_buffer {
	uint32_t head, tail;
	char data[CONNECTION_BUFFER_SIZE];
	uint16_t capacity, head, tail;
	char *data;
};

struct connection {
	struct connection_buffer in, out;
	struct connection_buffer fds_in, fds_out;
	char connection_data[CONNECTION_BUFFER_SIZE * 2 + CONNECTION_FD_BUFFER_SIZE * 2];

	int fd;
	bool want_flush;
};

void connection_init(struct connection *connection);

int connection_read(struct connection *connection);
int connection_flush(struct connection *connection);


M include/protocol.h => include/protocol.h +1 -1
@@ 1,7 1,7 @@
#ifndef _SEATD_CONSTANTS_H
#define _SEATD_CONSTANTS_H

#define MAX_PATH_LEN	 256
#define MAX_PATH_LEN	 128
#define MAX_SEAT_LEN	 64
#define MAX_SEAT_DEVICES 128
#define MAX_SESSION_LEN	 64

M libseat/backend/seatd.c => libseat/backend/seatd.c +1 -0
@@ 276,6 276,7 @@ static struct libseat *_open_seat(struct libseat_seat_listener *listener, void *
	backend->connection.fd = fd;
	backend->base.impl = &seatd_impl;
	list_init(&backend->pending_events);
	connection_init(&backend->connection);

	struct proto_header header = {
		.opcode = CLIENT_OPEN_SEAT,

M seatd/client.c => seatd/client.c +1 -0
@@ 68,6 68,7 @@ struct client *client_create(struct server *server, int client_fd) {
	client->server = server;
	client->connection.fd = client_fd;
	list_init(&client->devices);
	connection_init(&client->connection);
	return client;
}