~hedgepigdaniel/dewobble

5f8f16d53138a178d8e6369d2c8a0ad42003f214 — Daniel Playfair Cal 2 years ago 357eef0 feat/input_image
make it compile, update C bindings
M include/c_bindings/filter.h => include/c_bindings/filter.h +32 -14
@@ 11,6 11,7 @@ extern "C" {
#include <CL/cl.h>

#include "filter_config.h"
#include "frame.h"

/**
 * @copydoc dewobble::FilterBase


@@ 36,14 37,27 @@ void dewobble_filter_destroy(DewobbleFilter *filter);

/**
 * Push an input frame into the filter
 * @param input_buffer OpenCL buffer containing input frame in NV12 format. The
 * buffer should not be released until after the output frame is consumed.
 * @param extra Opaque pointer to extra data for this frame
 * @param frame DewobbleFrame containing the input frame
 */
int dewobble_filter_push_input_frame(
    DewobbleFilter c_filter,
    cl_mem input_buffer,
    void *extra);
    DewobbleFrame frame);

/**
 * Whether there is at least one input frame which is no longer required by
 * the filter
 * @return true if there is a frame that can be unreferenced
 */
int dewobble_filter_can_unref_input_frame(DewobbleFilter c_filter);

/**
 * Push an input frame into the filter
 * @param frame pointer to a DewobbleFrame which will point to the unreferenced
 * input frame on success
 */
int dewobble_filter_unref_input_frame(
    DewobbleFilter c_filter,
    DewobbleFrame *frame);

/**
 * Notify the filter that there will not be more input frames (and therefore


@@ 52,24 66,28 @@ int dewobble_filter_push_input_frame(
int dewobble_filter_end_input(DewobbleFilter c_filter);

/**
 * Provide an empty frame to be used for output
 * @param frame the frame. A reference will be kept to the frame until it is
 * returned from @ref dewobble_filter_pull_output_frame.
 */
int dewobble_filter_provide_output_frame(
    DewobbleFilter c_filter,
    DewobbleFrame frame);

/**
 * Determine if there is at least one output frame ready (i.e. if a call to
 * pull_frame would succeed)
 * dewobble_filter_pull_output_frame would succeed)
 */
int dewobble_filter_output_frame_ready(DewobbleFilter c_filter);

/**
 * Pull an output frame from the filter
 * @param output_buffer Pointer to an OpenCL buffer which will be set to point
 * to the output frame
 * @param output_buffer Pointer to an OpenCL buffer which will be set to the
 * buffer passed as the input frame
 * @param extra will be set to the opaque pointer to extra data about this frame
 * @param frame A pointer to a DewobbleFrame which will point to the output
 * frame on success
 */
int dewobble_filter_pull_output_frame(
    DewobbleFilter c_filter,
    cl_mem *output_buffer,
    cl_mem *input_buffer,
    void **extra);
    DewobbleFrame *frame);

#ifdef __cplusplus
}

M include/c_bindings/filter_config.h => include/c_bindings/filter_config.h +16 -2
@@ 36,6 36,13 @@ typedef enum DewobbleBorderType {
    DEWOBBLE_NB_BORDER_TYPES,
} DewobbleBorderType;

typedef enum DewobbleInputFramePersistence {
    DEWOBBLE_INPUT_FRAME_PERSISTENCE_NONE,
    DEWOBBLE_INPUT_FRAME_PERSISTENCE_ANALYSIS,
    DEWOBBLE_INPUT_FRAME_PERSISTENCE_TRANSFORMATION,
    DEWOBBLE_NB_FRAME_PERSISTENCE_TYPES,
} DewobbleInputFramePersistence;

/**
 * @copydoc dewobble::FilterConfig
 */


@@ 59,14 66,14 @@ void dewobble_filter_config_destroy(DewobbleFilterConfig *filter);
 */
void dewobble_filter_config_set_interpolation(
    DewobbleFilterConfig c_config,
    int interpolation);
    DewobbleInterpolation interpolation);

/**
 * Set border extrapolation algorithm
 */
void dewobble_filter_config_set_border_type(
    DewobbleFilterConfig c_config,
    int border_type);
    DewobbleBorderType border_type);

/**
 * Set border colour (for fixed border)


@@ 94,6 101,13 @@ void dewobble_filter_config_set_opencl_device(
    DewobbleFilterConfig c_config,
    cl_device_id device);

/**
 * Set the persistence model for input frames
 */
void dewobble_filter_config_set_input_frame_persistence(
    DewobbleFilterConfig c_config,
    DewobbleInputFramePersistence persistence);

#ifdef __cplusplus
}
#endif

A include/c_bindings/frame.h => include/c_bindings/frame.h +50 -0
@@ 0,0 1,50 @@
#ifndef DEWOBBLE_FRAME_H
#define DEWOBBLE_FRAME_H

/**
 * @file
 */

#ifdef __cplusplus
extern "C" {
#endif
#include <CL/cl.h>

typedef enum DewobblePixelFormat {
    DEWOBBLE_PIXEL_FORMAT_NV12_SINGLE_PLANE,
    DEWOBBLE_NB_PIXEL_FORMATS,
} DewobblePixelFormat;

typedef enum DewobbleMemoryType {
    DEWOBBLE_MEMORY_OPENCL_BUFFER,
    DEWOBBLE_MEMORY_OPENCL_IMAGE,
    DEWOBBLE_NB_MEMORY_TYPES,
} DewobbleMemoryType;

/**
 * @copydoc dewobble::Frame
 */
typedef struct _DewobbleFrame *DewobbleFrame;

DewobbleFrame dewobble_frame_create_from_nv12_single_opencl_buffer(
    cl_mem buffer,
    void *extra,
    int crop_left,
    int crop_right,
    int crop_top,
    int crop_bottom);

void dewobble_frame_get_opencl_planes(
    DewobbleFrame frame,
    cl_mem *result,
    int max_planes);

void *dewobble_frame_get_extra(DewobbleFrame frame);

void dewobble_frame_destroy(DewobbleFrame *c_frame);

#ifdef __cplusplus
}
#endif

#endif // DEWOBBLE_FRAME_H

M include/filter_sync.hpp => include/filter_sync.hpp +1 -1
@@ 64,7 64,7 @@ class FilterSync : public FilterBase
    void flush_frames();
    void init_after_changed_opencl_context();
    IntermediateFrame get_intermediate_frame(Frame frame);
    cv::UMat get_bgr_for_frame(Frame frame);
    cv::UMat get_bgr_for_frame(StabilizedFrame stabilized_frame);
    void write_output_frame(cv::UMat warped_frame, Frame frame);

  public:

M include/frame.hpp => include/frame.hpp +4 -4
@@ 7,7 7,7 @@ namespace dewobble
{

enum PixelFormat {
    DEWOBBLE_PIXEL_FORMAT_NV12,
    DEWOBBLE_PIXEL_FORMAT_NV12_SINGLE_PLANE,
};

enum MemoryType {


@@ 42,14 42,14 @@ class Frame

    const MemoryType memory_type = DEWOBBLE_MEMORY_OPENCL_IMAGE;

    const PixelFormat pixel_format = DEWOBBLE_PIXEL_FORMAT_NV12;
    const PixelFormat pixel_format = DEWOBBLE_PIXEL_FORMAT_NV12_SINGLE_PLANE;

    const std::vector<cl_mem> opencl_planes;
    const std::vector<void *> memory_planes;

    const void *extra;

    static Frame create_from_opencl_images_nv12(
    static Frame create_from_nv12_opencl_images(
        cl_mem luma,
        cl_mem chroma,
        void *extra = NULL,


@@ 58,7 58,7 @@ class Frame
        int crop_top = 0,
        int crop_bottom = 0);

    static Frame create_from_opencl_single_buffer_nv12(
    static Frame create_from_nv12_single_opencl_buffer(
        cl_mem buffer,
        void *extra = NULL,
        int crop_left = 0,

M include/intermediate_frame.hpp => include/intermediate_frame.hpp +1 -2
@@ 12,10 12,9 @@ class IntermediateFrame
  public:
    cv::UMat umat;
    Frame input_frame;
    void *extra;
    const int width;
    const int height;
    IntermediateFrame(cv::UMat umat, Frame input_frame, void *extra);
    IntermediateFrame(cv::UMat umat, Frame input_frame);
};
} // namespace dewobble


M include/rotation_detector.hpp => include/rotation_detector.hpp +5 -0
@@ 3,6 3,7 @@

#include <list>
#include <opencv2/core.hpp>
#include <queue>
#include <set>

#include "camera.hpp"


@@ 72,6 73,8 @@ class RotationDetector
    // Whether the input has ended
    bool m_input_ended = false;

    std::queue<Frame> m_unreferenced_input_frames;

    std::set<int> guess_camera_rotation(
        std::vector<cv::Point2f> points_prev,
        std::vector<cv::Point2f> points_current,


@@ 83,6 86,8 @@ class RotationDetector
    void end_input();
    FrameWithOrientation pull_frame();
    bool frame_ready();
    Frame unref_input_frame();
    bool can_unref_input_frame();
};
} // namespace dewobble


M src/c_bindings/filter.cpp => src/c_bindings/filter.cpp +56 -31
@@ 45,21 45,33 @@ void dewobble_filter_destroy(DewobbleFilter *c_filter)
    }
}

// int dewobble_filter_push_input_frame(
//     DewobbleFilter c_filter,
//     cl_mem input_buffer,
//     void *extra)
// {
//     shared_ptr<dewobble::FilterBase> filter =
//         *((shared_ptr<dewobble::FilterBase> *)c_filter);
//     int err = 0;
//     try {
//         filter->push_input_frame(input_buffer, extra);
//     } catch (...) {
//         err = -1;
//     }
//     return err;
// }
int dewobble_filter_push_input_frame(
    DewobbleFilter c_filter,
    DewobbleFrame c_frame)
{
    shared_ptr<dewobble::FilterBase> filter =
        *((shared_ptr<dewobble::FilterBase> *)c_filter);
    shared_ptr<dewobble::Frame> frame =
        *((shared_ptr<dewobble::Frame> *)c_frame);
    int err = 0;
    try {
        filter->push_input_frame(*frame);
    } catch (...) {
        err = -1;
    }
    return err;
}

int dewobble_filter_can_unref_input_frame(DewobbleFilter c_filter)
{
    shared_ptr<dewobble::FilterBase> filter =
        *((shared_ptr<dewobble::FilterBase> *)c_filter);
    return filter->can_unref_input_frame();
}

int dewobble_filter_unref_input_frame(
    DewobbleFilter c_filter,
    DewobbleFrame *frame);

int dewobble_filter_end_input(DewobbleFilter c_filter)
{


@@ 74,6 86,18 @@ int dewobble_filter_end_input(DewobbleFilter c_filter)
    return err;
}

int dewobble_filter_provide_output_frame(
    DewobbleFilter c_filter,
    DewobbleFrame c_frame)
{
    shared_ptr<dewobble::FilterBase> filter =
        *((shared_ptr<dewobble::FilterBase> *)c_filter);
    shared_ptr<dewobble::Frame> frame =
        *((shared_ptr<dewobble::Frame> *)c_frame);
    filter->provide_output_frame(*frame);
    return 0;
}

int dewobble_filter_output_frame_ready(DewobbleFilter c_filter)
{
    shared_ptr<dewobble::FilterBase> filter =


@@ 81,19 105,20 @@ int dewobble_filter_output_frame_ready(DewobbleFilter c_filter)
    return filter->output_frame_ready();
}

// int dewobble_filter_pull_output_frame(
//     DewobbleFilter c_filter,
//     cl_mem *output_buffer,
//     cl_mem *input_buffer,
//     void **extra)
// {
//     shared_ptr<dewobble::FilterBase> filter =
//         *((shared_ptr<dewobble::FilterBase> *)c_filter);
//     int err = 0;
//     try {
//         filter->pull_frame(output_buffer, input_buffer, extra);
//     } catch (...) {
//         err = -1;
//     }
//     return err;
// }
int dewobble_filter_pull_output_frame(
    DewobbleFilter c_filter,
    DewobbleFrame *c_frame)
{
    shared_ptr<dewobble::FilterBase> filter =
        *((shared_ptr<dewobble::FilterBase> *)c_filter);
    shared_ptr<dewobble::Frame> **result =
        ((shared_ptr<dewobble::Frame> **)c_frame);
    int err = 0;
    try {
        dewobble::Frame frame = filter->pull_output_frame();
        *result = new shared_ptr<dewobble::Frame>(new dewobble::Frame(frame));
    } catch (...) {
        err = -1;
    }
    return err;
}

M src/c_bindings/filter_config.cpp => src/c_bindings/filter_config.cpp +13 -2
@@ 38,7 38,7 @@ void dewobble_filter_config_destroy(DewobbleFilterConfig *c_config)

void dewobble_filter_config_set_interpolation(
    DewobbleFilterConfig c_config,
    int interpolation)
    DewobbleInterpolation interpolation)
{
    shared_ptr<dewobble::FilterConfig> config =
        *((shared_ptr<dewobble::FilterConfig> *)c_config);


@@ 48,7 48,7 @@ void dewobble_filter_config_set_interpolation(

void dewobble_filter_config_set_border_type(
    DewobbleFilterConfig c_config,
    int border_type)
    DewobbleBorderType border_type)
{
    shared_ptr<dewobble::FilterConfig> config =
        *((shared_ptr<dewobble::FilterConfig> *)c_config);


@@ 99,4 99,15 @@ void dewobble_filter_config_set_opencl_device(
        *((shared_ptr<dewobble::FilterConfig> *)c_config);

    config->set_opencl_device(device);
}

void dewobble_filter_config_set_input_frame_persistence(
    DewobbleFilterConfig c_config,
    DewobbleInputFramePersistence persistence)
{
    shared_ptr<dewobble::FilterConfig> config =
        *((shared_ptr<dewobble::FilterConfig> *)c_config);

    config->set_input_frame_persistence(
        (dewobble::InputFramePersistence)persistence);
}
\ No newline at end of file

A src/c_bindings/frame.cpp => src/c_bindings/frame.cpp +60 -0
@@ 0,0 1,60 @@
#include <memory>
#include <stdio.h>

#include "c_bindings/frame.h"
#include "frame.hpp"

using namespace std;

struct _DewobbleFrame {
};

DewobbleFrame dewobble_frame_create_from_nv12_single_opencl_buffer(
    cl_mem buffer,
    void *extra,
    int crop_left,
    int crop_right,
    int crop_top,
    int crop_bottom)
{
    shared_ptr<dewobble::Frame> *frame;
    try {
        frame = new shared_ptr<dewobble::Frame>(
            dewobble::Frame::create_from_nv12_single_opencl_buffer(
                buffer, extra, crop_left, crop_right, crop_top, crop_bottom));
    } catch (...) {
        frame = NULL;
    }
    return (DewobbleFrame)frame;
}

void dewobble_frame_destroy(DewobbleFrame *c_frame)
{
    shared_ptr<dewobble::Frame> **frame =
        (shared_ptr<dewobble::Frame> **)c_frame;
    if (frame != NULL) {
        delete *frame;
        *frame = NULL;
    }
}

void dewobble_frame_get_opencl_planes(
    DewobbleFrame c_frame,
    cl_mem *result,
    int max_planes)
{
    shared_ptr<dewobble::Frame> *frame = (shared_ptr<dewobble::Frame> *)c_frame;
    vector<cl_mem> planes = (*frame)->opencl_planes;
    for (int i = 0; i < planes.size(); i++) {
        if (i > max_planes) {
            break;
        }
        result[i] = planes[i];
    }
}

void *dewobble_frame_get_extra(DewobbleFrame c_frame)
{
    shared_ptr<dewobble::Frame> *frame = (shared_ptr<dewobble::Frame> *)c_frame;
    return (*frame)->extra;
}

M src/filter/filter_sync.cpp => src/filter/filter_sync.cpp +57 -40
@@ 304,16 304,26 @@ void FilterSync::add_debug_info(UMat output, StabilizedFrame frame)

IntermediateFrame FilterSync::get_intermediate_frame(Frame frame)
{
    throw "TODO";
    // UMat input_frame;
    // ocl::convertFromBuffer(
    //     input_buffer,
    //     m_config.get_input_camera().m_width,
    //     m_config.get_input_camera().m_height * 3 / 2,
    //     m_config.get_input_camera().m_width,
    //     CV_8U,
    //     input_frame);
    // return IntermediateFrame(input_frame, extra);
    if (frame.memory_type != DEWOBBLE_MEMORY_OPENCL_BUFFER ||
        frame.pixel_format != DEWOBBLE_PIXEL_FORMAT_NV12_SINGLE_PLANE) {
        throw "Not implemented";
    }
    UMat input_frame;
    ocl::convertFromBuffer(
        frame.opencl_planes[0],
        frame.width,
        frame.height,
        frame.width,
        CV_8U,
        input_frame);
    UMat cropped_input_frame(
        input_frame,
        Rect(
            frame.crop_left,
            frame.crop_top,
            frame.width - frame.crop_right,
            frame.height - frame.crop_bottom));
    return IntermediateFrame(cropped_input_frame, frame);
}

void FilterSync::push_input_frame(Frame frame)


@@ 351,58 361,65 @@ bool FilterSync::output_frame_ready()
           m_empty_output_frames.size() > 0;
}

UMat FilterSync::get_bgr_for_frame(Frame frame)
UMat FilterSync::get_bgr_for_frame(StabilizedFrame frame)
{
    // cvtColorNv12oBgr(frame.frame, m_frame_bgr);
    // return m_frame_bgr;
    throw "TODO";
    cvtColorNv12oBgr(frame.umat, m_frame_bgr);
    return m_frame_bgr;
}

void FilterSync::write_output_frame(UMat warped_frame, Frame output_frame)
{
    // ocl::OpenCLExecutionContext context =
    //     ocl::OpenCLExecutionContext::getCurrentRef();

    // UMat output_frame_yuv;
    // if (*output_buffer == NULL) {
    //     *output_buffer = get_output_frame_buffer();
    // }
    // ocl::convertFromBuffer(
    //     *output_buffer,
    //     m_config.get_output_camera().m_width,
    //     m_config.get_output_camera().m_height * 3 / 2,
    //     m_config.get_output_camera().m_width,
    //     CV_8U,
    //     output_frame_yuv);
    // cvtColorBgrToNv12(warped_frame, output_frame_yuv);
    // if (extra != NULL) {
    //     *extra = frame.extra;
    // }

    // *input_buffer = (cl_mem)frame.frame.handle(ACCESS_READ);
    throw "TODO";
    if (output_frame.memory_type != DEWOBBLE_MEMORY_OPENCL_BUFFER ||
        output_frame.pixel_format != DEWOBBLE_PIXEL_FORMAT_NV12_SINGLE_PLANE) {
        throw "Not implemented";
    }

    ocl::OpenCLExecutionContext context =
        ocl::OpenCLExecutionContext::getCurrentRef();

    UMat output_frame_yuv;
    ocl::convertFromBuffer(
        output_frame.opencl_planes[0],
        output_frame.width,
        output_frame.height,
        output_frame.width,
        CV_8U,
        output_frame_yuv);

    UMat output_frame_yuv_cropped(
        output_frame_yuv,
        Rect(
            output_frame.crop_left,
            output_frame.crop_top,
            output_frame.width - output_frame.crop_right,
            output_frame.height - output_frame.crop_bottom));

    cvtColorBgrToNv12(warped_frame, output_frame_yuv_cropped);
}

Frame FilterSync::pull_output_frame()
{
    StabilizedFrame frame = m_intermediate_output_frames.front();
    StabilizedFrame stabilized_frame = m_intermediate_output_frames.front();
    m_intermediate_output_frames.pop();

    Frame output_frame = m_empty_output_frames.front();
    m_empty_output_frames.pop();

    UMat frame_bgr; // = get_bgr_for_frame(frame); TODO
    UMat frame_bgr = get_bgr_for_frame(stabilized_frame);

    warp_frame(m_frame_bgr, m_warped_frame, frame.corrective_rotation.inv());
    warp_frame(
        m_frame_bgr,
        m_warped_frame,
        stabilized_frame.corrective_rotation.inv());

    if (m_config.get_debug()) {
        add_debug_info(m_warped_frame, frame);
        add_debug_info(m_warped_frame, stabilized_frame);
    }

    write_output_frame(m_warped_frame, output_frame);
    if (m_config.get_input_frame_persistence() ==
        DEWOBBLE_INPUT_FRAME_PERSISTENCE_TRANSFORMATION) {
        unreference_input_frame(frame.input_frame);
        unreference_input_frame(stabilized_frame.input_frame);
    }
    return output_frame;
}

M src/intermediate_frame.cpp => src/intermediate_frame.cpp +2 -2
@@ 4,8 4,8 @@ using namespace dewobble;
using namespace cv;
using namespace std;

IntermediateFrame::IntermediateFrame(UMat umat, Frame input_frame, void *extra)
    : umat(umat), input_frame(input_frame), extra(extra),
IntermediateFrame::IntermediateFrame(UMat umat, Frame input_frame)
    : umat(umat), input_frame(input_frame),
      width(input_frame.width - input_frame.crop_left - input_frame.crop_right),
      height(
          input_frame.height - input_frame.crop_top - input_frame.crop_bottom)

M src/rotation_detector.cpp => src/rotation_detector.cpp +12 -0
@@ 287,3 287,15 @@ bool RotationDetector::frame_ready()
    return (m_input_ended && m_buffered_frames.size() > 0) ||
           (m_buffered_frames.size() >= m_interpolation_horizon);
}

Frame RotationDetector::unref_input_frame()
{
    Frame frame = m_unreferenced_input_frames.front();
    m_unreferenced_input_frames.pop();
    return frame;
}

bool RotationDetector::can_unref_input_frame()
{
    return m_unreferenced_input_frames.size() > 0;
}
\ No newline at end of file

M src/stabilizer/stabilizer_fixed.cpp => src/stabilizer/stabilizer_fixed.cpp +3 -0
@@ 17,6 17,9 @@ void StabilizerFixed::flush_frames()
        m_buffered_frames.push(StabilizedFrame(
            oriented, oriented.orientation.inv(), oriented.debug_info));
    }
    while (m_rotation_detector.can_unref_input_frame()) {
        m_frames_to_unref.push(m_rotation_detector.unref_input_frame());
    }
}

void StabilizerFixed::end_input()

M src/stabilizer/stabilizer_savitzky_golay.cpp => src/stabilizer/stabilizer_savitzky_golay.cpp +3 -0
@@ 50,6 50,9 @@ void StabilizerSavitzkyGolay::try_pull_frames_from_detector()
        m_buffered_frames.push(oriented);
        m_rotation_filter.add(eigen_mat_from_cv_mat(oriented.orientation));
    }
    while (m_rotation_detector.can_unref_input_frame()) {
        m_frames_to_unref.push(m_rotation_detector.unref_input_frame());
    }
}

void StabilizerSavitzkyGolay::end_input()