~emersion/vaapi-decoder

cd01be8cdfd82d3e2704d0e7d3d414e52fa0e439 — Simon Ser 8 months ago d5167c5
Get modifier list from linux-dmabuf

This allows us to stop hardcoding a static list of modifiers.
1 files changed, 56 insertions(+), 12 deletions(-)

M main.c
M main.c => main.c +56 -12
@@ 31,6 31,9 @@ static struct wl_surface *surface = NULL;
#define MAX_FRAMES_IN_FLIGHT 10
static int frames_in_flight = 0;

static uint64_t modifiers[64];
static size_t modifiers_len = 0;

static const struct {
	uint32_t format;
	int nb_layers;


@@ 72,17 75,7 @@ static int check_hw_device_type(enum AVHWDeviceType type) {
	return -1;
}

// Allow-list of AMD modifiers suitable for direct NV12 scan-out
// TODO: in the future we should get this from the linux-dmabuf Wayland protocol
uint64_t modifiers[] = {
	0x200000000000901, // AMD(TILE_VERSION = GFX9, TILE = GFX9_64K_S)
	0x200000000000a01, // AMD(TILE_VERSION = GFX9, TILE = GFX9_64K_D)
	0x200000000401901, // AMD(TILE_VERSION = GFX9, TILE = GFX9_64K_S_X, PIPE_XOR_BITS = 2, BANK_XOR_BITS = 0)
};
VADRMFormatModifierList vaapi_modifier_list = {
	.num_modifiers = sizeof(modifiers) / sizeof(modifiers[0]),
	.modifiers = modifiers,
};
VADRMFormatModifierList vaapi_modifier_list = {0};
VASurfaceAttrib vaapi_surface_attrib = {
	.type = VASurfaceAttribDRMFormatModifiers,
	.flags = VA_SURFACE_ATTRIB_SETTABLE,


@@ 207,7 200,7 @@ static void handle_global(void *data, struct wl_registry *registry,
			wl_registry_bind(registry, name, &gamescope_xwayland_interface, 1);
	} else if (strcmp(interface, zwp_linux_dmabuf_v1_interface.name) == 0) {
		zwp_linux_dmabuf_v1 =
			wl_registry_bind(registry, name, &zwp_linux_dmabuf_v1_interface, 2);
			wl_registry_bind(registry, name, &zwp_linux_dmabuf_v1_interface, 3);
	}
}



@@ 285,6 278,43 @@ static const struct wl_callback_listener frame_callback_listener = {
	.done = handle_frame_callback_done,
};

static void handle_dmabuf_format(void *data, struct zwp_linux_dmabuf_v1 *linux_dmabuf,
		uint32_t format) {
	// Superseded by handle_dmabuf_modifier
}

static void handle_dmabuf_modifier(void *data, struct zwp_linux_dmabuf_v1 *linux_dmabuf,
		uint32_t format, uint32_t modifier_hi, uint32_t modifier_lo) {
	// TODO: accumulate modifiers for all formats, don't assume we'll get NV12
	if (format != DRM_FORMAT_NV12) {
		return;
	}

	uint64_t mod = ((uint64_t)modifier_hi << 32) | modifier_lo;

	// We want an explicit modifier, not INVALID which indicates an implicit
	// buffer tiling.
	if (mod == DRM_FORMAT_MOD_INVALID) {
		return;
	}

	// Ignore LINEAR, because AMD hardware can't directly scan-out rotated
	// LINEAR buffers.
	if (mod == DRM_FORMAT_MOD_LINEAR) {
		return;
	}

	modifiers[modifiers_len] = mod;
	modifiers_len++;

	fprintf(stderr, "Supported NV12 modifier: 0x%"PRIX64"\n", mod);
}

static const struct zwp_linux_dmabuf_v1_listener zwp_linux_dmabuf_v1_listener = {
	.format = handle_dmabuf_format,
	.modifier = handle_dmabuf_modifier,
};

static void push_frame(void) {
	AVFrame *frame = NULL;
	int ret = decode_frame(input_ctx, video_stream, decoder_ctx, &frame);


@@ 431,6 461,20 @@ int main(int argc, char *argv[]) {
		return 1;
	}

	zwp_linux_dmabuf_v1_add_listener(zwp_linux_dmabuf_v1, &zwp_linux_dmabuf_v1_listener, NULL);

	// Second roundtrip to get DMA-BUF format modifiers
	wl_display_roundtrip(display);

	if (modifiers_len == 0) {
		fprintf(stderr, "zwp_linux_dmabuf_v1 doesn't support any modifier\n");
		return 1;
	}

	// Fill our VA-API modifier allow-list
	vaapi_modifier_list.num_modifiers = modifiers_len;
	vaapi_modifier_list.modifiers = modifiers;

	xcb_connection_t *conn = xcb_connect(NULL, NULL);
	if (!conn || xcb_connection_has_error(conn)) {
		fprintf(stderr, "Failed to connect to X11 display\n");