~emersion/vaapi-decoder

c2c800f58f11bd41d1e312f52827f2e3f34ad398 — Simon Ser 9 months ago bc7f7b0 x11-hack
hacky hack

Juts pretend this never happened.
2 files changed, 123 insertions(+), 1 deletions(-)

M Makefile
M main.c
M Makefile => Makefile +1 -1
@@ 1,6 1,6 @@
CFLAGS = -g -Wall -Wextra -Wno-unused-parameter

deps = libavcodec libavformat libavutil libdrm
deps = libavcodec libavformat libavutil libdrm xcb xcb-dri3 xcb-present
depflags = $(shell pkg-config $(deps) --cflags --libs)

all: vaapi-decoder

M main.c => main.c +122 -0
@@ 8,6 8,23 @@
#include <libavutil/hwcontext.h>
#include <libavutil/hwcontext_drm.h>
#include <libavutil/pixdesc.h>
#include <xcb/xcb.h>
#include <xcb/dri3.h>
#include <xcb/present.h>

#define GAMESCOPE_DMABUF_PROP "GAMESCOPE_DMABUF"

struct gamescope_dmabuf_plane {
	uint32_t stride;
	uint32_t offset;
};

struct gamescope_dmabuf {
	uint32_t format;
	uint64_t modifier;
	uint32_t n_planes;
	struct gamescope_dmabuf_plane planes[4];
};

static enum AVPixelFormat hw_pix_fmt = AV_PIX_FMT_NONE;



@@ 116,6 133,27 @@ static int decode_frame(AVFormatContext *input_ctx, int video_stream,
	return AVERROR_EOF;
}

static xcb_depth_t *get_depth(xcb_screen_t *screen, uint8_t depth) {
	xcb_depth_iterator_t iter = xcb_screen_allowed_depths_iterator(screen);
	while (iter.rem > 0) {
		if (iter.data->depth == depth) {
			return iter.data;
		}
		xcb_depth_next(&iter);
	}
	return NULL;
}

static xcb_visualid_t pick_visualid(xcb_depth_t *depth) {
	xcb_visualtype_t *visuals = xcb_depth_visuals(depth);
	for (int i = 0; i < xcb_depth_visuals_length(depth); i++) {
		if (visuals[i]._class == XCB_VISUAL_CLASS_TRUE_COLOR) {
			return visuals[i].visual_id;
		}
	}
	return 0;
}

int main(int argc, char *argv[]) {
	if (argc < 2) {
		fprintf(stderr, "usage: %s <file>\n", argv[0]);


@@ 196,6 234,40 @@ int main(int argc, char *argv[]) {
		return 1;
	}

	xcb_connection_t *conn = xcb_connect(NULL, NULL);
	if (!conn || xcb_connection_has_error(conn)) {
		fprintf(stderr, "Failed to connect to X11 display\n");
		return 1;
	}

	xcb_screen_t *screen = xcb_setup_roots_iterator(xcb_get_setup(conn)).data;

	xcb_depth_t *depth = get_depth(screen, 24);
	if (!depth) {
		fprintf(stderr, "Failed to find depth\n");
		return 1;
	}
	xcb_visualid_t visualid = pick_visualid(depth);
	if (!visualid) {
		fprintf(stderr, "Failed to find visualid\n");
		return 1;
	}

	xcb_colormap_t colormap = xcb_generate_id(conn);
	xcb_create_colormap(conn, XCB_COLORMAP_ALLOC_NONE, colormap, screen->root, visualid);

	uint32_t mask = XCB_CW_BORDER_PIXEL | XCB_CW_COLORMAP;
	uint32_t values[] = { 0, colormap };

	xcb_window_t win = xcb_generate_id(conn);
	xcb_create_window(conn, depth->depth, win, screen->root, 0, 0, 1920, 1080, 0, XCB_WINDOW_CLASS_INPUT_OUTPUT, visualid, mask, values);
	xcb_map_window(conn, win);
	xcb_flush(conn);

	xcb_intern_atom_cookie_t cookie = xcb_intern_atom(conn, true, strlen(GAMESCOPE_DMABUF_PROP), GAMESCOPE_DMABUF_PROP);
	xcb_intern_atom_reply_t *reply = xcb_intern_atom_reply(conn, cookie, NULL);
	xcb_atom_t gamescope_dmabuf_prop = reply->atom;

	AVFrame *frame = NULL;
	ret = decode_frame(input_ctx, video_stream, decoder_ctx, &frame);
	if (ret < 0) {


@@ 244,6 316,48 @@ int main(int argc, char *argv[]) {
	}
	fprintf(stderr, "DRM format: 0x%X\n", drm_format);

	if (drm_format != DRM_FORMAT_NV12) {
		fprintf(stderr, "Only NV12 is supported\n");
		return 1;
	}

	int fd = dup(drm_frame_desc->objects[0].fd);
	uint32_t size = lseek(fd, 0, SEEK_END);
	uint8_t frame_depth = 16;
	uint8_t frame_bpp = 16;
	uint16_t stride = drm_frame_desc->layers[0].planes[0].pitch;

	struct gamescope_dmabuf gamescope_dmabuf = {
		.format = drm_format,
		.modifier = drm_frame_desc->objects[0].format_modifier,
	};
	int k = 0;
	for (int i = 0; i < drm_frame_desc->nb_layers; i++) {
		AVDRMLayerDescriptor *drm_layer = &drm_frame_desc->layers[i];
		for (int j = 0; j < drm_layer->nb_planes; j++) {
			AVDRMPlaneDescriptor *drm_plane = &drm_layer->planes[j];
			gamescope_dmabuf.planes[k] = (struct gamescope_dmabuf_plane){
				.offset = drm_plane->offset,
				.stride = drm_plane->pitch,
			};
			k++;
		}
	}
	gamescope_dmabuf.n_planes = k;

	xcb_change_property(conn, XCB_PROP_MODE_REPLACE, win, gamescope_dmabuf_prop, XCB_ATOM_NONE, 8, sizeof(gamescope_dmabuf), &gamescope_dmabuf);

	xcb_pixmap_t pixmap = xcb_generate_id(conn);
	xcb_dri3_pixmap_from_buffer(conn, pixmap, win, size, drm_frame->width,
		drm_frame->height, stride, frame_depth, frame_bpp, fd);

	uint32_t serial = 0;
	uint32_t options = 0;
	uint64_t target_msc = 0;
	xcb_present_pixmap(conn, win, pixmap, serial, 0, XCB_NONE, 0, 0, XCB_NONE, XCB_NONE, XCB_NONE, options, target_msc, 0, 0, 0, NULL);

	xcb_flush(conn);

	for (int i = 0; i < drm_frame_desc->nb_objects; i++) {
		close(drm_frame_desc->objects[i].fd);
		drm_frame_desc->objects[i].fd = -1;


@@ 252,6 366,14 @@ int main(int argc, char *argv[]) {
	av_frame_free(&drm_frame);
	av_frame_free(&frame);

	while (1) {
		xcb_generic_event_t *event = xcb_wait_for_event(conn);
		if (!event) {
			break;
		}
		free(event);
	}

	avcodec_free_context(&decoder_ctx);
	avformat_close_input(&input_ctx);
	av_buffer_unref(&hw_device_ctx);