~hedgepigdaniel/dewobble

de436b63eef4e3fff47bca59e989d5e521ed2604 — Daniel Playfair Cal 2 years ago 2b6b10c
docs: generally fill out docs
M README.md => README.md +0 -2
@@ 4,8 4,6 @@ Dewobble is a library for video motion stabilization and camera projection chang

It is named dewobble because its accurate camera models avoid the wobbling effect that is produced by many other video stabilisation libraries when they are applied to videos with a wide field of view.

<img alt="" src="https://git.sr.ht/~hedgepigdaniel/dewobble/blob/trunk/logo.png" width="320px" />

## Documentation

See the detailed Doxygen [documentation](https://www.danielplayfaircal.com/dewobble/index.html).

M include/c_bindings/camera.h => include/c_bindings/camera.h +16 -0
@@ 1,19 1,32 @@
#ifndef DEWOBBLE_CAMERA_H
#define DEWOBBLE_CAMERA_H

/**
 * @file
 */

#ifdef __cplusplus
extern "C" {
#endif
#include <CL/cl.h>

/**
 * @copydoc dewobble::Camera
 */
typedef struct _DewobbleCamera *DewobbleCamera;

/**
 * Camera projection model.
 */
typedef enum DewobbleProjection {
    DEWOBBLE_PROJECTION_RECTILINEAR,
    DEWOBBLE_PROJECTION_EQUIDISTANT_FISHEYE,
    DEWOBBLE_NB_PROJECTIONS,
} DewobbleProjection;

/**
 * Create a camera. See also dewobble::Camera.
 */
DewobbleCamera dewobble_camera_create(
    DewobbleProjection projection,
    double diagonal_field_of_view,


@@ 22,6 35,9 @@ DewobbleCamera dewobble_camera_create(
    double focal_point_x,
    double focal_point_y);

/**
 * Destroy a camera
 */
void dewobble_camera_destroy(DewobbleCamera *camera);

#ifdef __cplusplus

M include/c_bindings/filter.h => include/c_bindings/filter.h +20 -3
@@ 1,6 1,10 @@
#ifndef DEWOBBLE_FILTER_H
#define DEWOBBLE_FILTER_H

/**
 * @file
 */

#ifdef __cplusplus
extern "C" {
#endif


@@ 9,15 13,25 @@ extern "C" {
#include "filter_config.h"

/**
 * A filter for video frames which can apply lens projection changes
 * and/or video stabilisation
 * @copydoc dewobble::FilterBase
 */
typedef struct _DewobbleFilter *DewobbleFilter;

/**
 * Create a filter which isolates OpenCV in a separate thread. See also
 * dewobble::FilterThreaded.
 */
DewobbleFilter dewobble_filter_create_threaded(DewobbleFilterConfig config);

/**
 * Create a filter which runs in the callers thread. See also
 * dewobble::FilterSync.
 */
DewobbleFilter dewobble_filter_create_sync(DewobbleFilterConfig config);

/**
 * Destroy a filter
 */
void dewobble_filter_destroy(DewobbleFilter *filter);

/**


@@ 45,7 59,8 @@ void dewobble_filter_release_input_frame_buffer(

/**
 * Push an input frame into the filter
 * @param input_buffer OpenCL buffer containing input frame in NV12 format
 * @param input_buffer OpenCL buffer containing input frame in NV12 format. The
 * buffer should not be released until after the output frame is consumed.
 * @param extra Opaque pointer to extra data for this frame
 */
int dewobble_filter_push_frame(


@@ 69,6 84,8 @@ int dewobble_filter_frame_ready(DewobbleFilter c_filter);
 * Pull an output frame from the filter
 * @param output_buffer Pointer to an OpenCL buffer which will be set to point
 * to the output frame
 * @param output_buffer Pointer to an OpenCL buffer which will be set to the
 * buffer passed as the input frame
 * @param extra will be set to the opaque pointer to extra data about this frame
 */
int dewobble_filter_pull_frame(

M include/c_bindings/filter_config.h => include/c_bindings/filter_config.h +35 -2
@@ 1,6 1,10 @@
#ifndef DEWOBBLE_FILTER_CONFIG_H
#define DEWOBBLE_FILTER_CONFIG_H

/**
 * @file
 */

#ifdef __cplusplus
extern "C" {
#endif


@@ 9,6 13,9 @@ extern "C" {
#include "camera.h"
#include "stabilizer.h"

/**
 * Pixel interpolation algorithm.
 */
typedef enum DewobbleInterpolation {
    DEWOBBLE_INTERPOLATION_NEAREST,
    DEWOBBLE_INTERPOLATION_LINEAR,


@@ 17,6 24,9 @@ typedef enum DewobbleInterpolation {
    DEWOBBLE_NB_INTERPOLATIONS,
} DewobbleInterpolation;

/**
 * Border extrapolation algorithm
 */
typedef enum DewobbleBorderType {
    DEWOBBLE_BORDER_CONSTANT,
    DEWOBBLE_BORDER_REPLICATE,


@@ 27,36 37,59 @@ typedef enum DewobbleBorderType {
} DewobbleBorderType;

/**
 * A filter for video frames which can apply lens projection changes
 * and/or video stabilisation
 * @copydoc dewobble::FilterConfig
 */
typedef struct _DewobbleFilterConfig *DewobbleFilterConfig;

/**
 * Create a Dewobble filter configuration. See dewobble::FilterConfig.
 */
DewobbleFilterConfig dewobble_filter_config_create(
    DewobbleCamera input_camera,
    DewobbleCamera output_camera,
    DewobbleStabilizer stabilizer);

/**
 * Destroy a filter configuration
 */
void dewobble_filter_config_destroy(DewobbleFilterConfig *filter);

/**
 * Set pixel interpolation algorithm
 */
void dewobble_filter_config_set_interpolation(
    DewobbleFilterConfig c_config,
    int interpolation);

/**
 * Set border extrapolation algorithm
 */
void dewobble_filter_config_set_border_type(
    DewobbleFilterConfig c_config,
    int border_type);

/**
 * Set border colour (for fixed border)
 */
void dewobble_filter_config_set_border_color(
    DewobbleFilterConfig c_config,
    double *border_colors);

/**
 * Set debug mode
 */
void dewobble_filter_config_set_debug(DewobbleFilterConfig c_config, int debug);

/**
 * Set OpenCL context to be used
 */
void dewobble_filter_config_set_opencl_context(
    DewobbleFilterConfig c_config,
    cl_context context);

/**
 * Set OpenCL device to be used
 */
void dewobble_filter_config_set_opencl_device(
    DewobbleFilterConfig c_config,
    cl_device_id device);

M include/c_bindings/stabilizer.h => include/c_bindings/stabilizer.h +20 -1
@@ 1,6 1,10 @@
#ifndef DEWOBBLE_STABILIZER_H
#define DEWOBBLE_STABILIZER_H

/**
 * @file
 */

#ifdef __cplusplus
extern "C" {
#endif


@@ 8,20 12,35 @@ extern "C" {

#include "camera.h"

/**
 * @copydoc dewobble::Stabilizer
 */
typedef struct _DewobbleStabilizer *DewobbleStabilizer;

/**
 * Create a null stabilizer. See also dewobble::StabilizerNone.
 */
DewobbleStabilizer dewobble_stabilizer_create_none();

/**
 * Create a null stabilizer. See also dewobble::StabilizerFixed.
 */
DewobbleStabilizer dewobble_stabilizer_create_fixed(
    DewobbleCamera camera,
    unsigned int interpolation_horizon);

/**
 * Create a null stabilizer. See also dewobble::StabilizerSavitzkyGolay.
 */
DewobbleStabilizer dewobble_stabilizer_create_savitzky_golay(
    DewobbleCamera camera,
    int radius,
    unsigned int interpolation_horizon);

void dewobble_stabilizer_destroy(DewobbleStabilizer *camera);
/**
 * Destroy a stabilizer
 */
void dewobble_stabilizer_destroy(DewobbleStabilizer *stabilizer);

#ifdef __cplusplus
}

M include/camera.hpp => include/camera.hpp +70 -0
@@ 6,19 6,50 @@

namespace dewobble
{

/**
 * Camera projection models.
 *
 * These define the type of relationship between real world coordinates and
 * projected coordinates in images captured by a camera.
 */
enum Projection {
    PROJECTION_RECTILINEAR,
    PROJECTION_EQUIDISTANT_FISHEYE,
};

/**
 * The properties of a camera which affect the mapping between pixels in images
 * and real world coordinates. May be used to interpret an input image or to
 * generate an output image.
 *
 */
class Camera
{
  public:
    /**
     * Projection model used for the camera
     */
    const Projection m_projection;
    /**
     * Focal length in pixels
     */
    const double m_focal_length;
    /**
     * Width of the image in pixels
     */
    const int m_width;
    /**
     * Height of the image in pixels
     */
    const int m_height;
    /**
     * Horizontal coordinate of the focal point, where the lens faces
     */
    const double m_focal_point_x;
    /**
     * Vertical coordinate of the focal point, where the lens faces
     */
    const double m_focal_point_y;
    Camera(
        Projection projection,


@@ 27,17 58,56 @@ class Camera
        int height,
        double focal_point_x,
        double focal_point_y);

    /**
     * Get the size of images produced by the camera
     *
     * @return cv::Size
     */
    cv::Size size() const;
    /**
     * Get the camera matrix which converts image points to world points
     *
     * @return cv::Matx33d
     */
    cv::Matx33d matrix() const;
    /**
     * Get the distortion coefficients for the OpenCV camera model
     *
     * @return cv::Mat
     */
    cv::Mat distortion_coefficients() const;
    /**
     * Convert a set of points in an image generated by the camera to points in
     * a chosen camera matrix.
     *
     * @param points The input points projected by this camera
     * @param rotation Optional rotation to apply to the points in the world
     * coordinate space
     * @param output_matrix Optional output camera matrix. If not given, the
     * identity matrix is used.
     * @return std::vector<cv::Point2f> The converted points projected by the
     * output matrix
     */
    std::vector<cv::Point2f> undistortPoints(
        std::vector<cv::Point2f> points,
        cv::InputArray rotation = cv::noArray(),
        cv::InputArray output_matrix = cv::noArray()) const;
    /**
     * Map a set of points using the specified input matrix to their location as
     * they would gbe projected by this camera.
     *
     * @param points The input points projected by the input matrix
     * @param input_matrix Optional input camera matrix. If not given, the
     * identity matrix is used.
     * @return std::vector<cv::Point2f> The output points projected by this
     * camera
     */
    std::vector<cv::Point2f> distortPoints(
        std::vector<cv::Point2f> points,
        cv::InputArray input_matrix = cv::noArray()) const;
};

} // namespace dewobble

#endif // DEWOBBLE_CAMERA_HPP

M include/filter_base.hpp => include/filter_base.hpp +6 -2
@@ 11,8 11,9 @@ namespace dewobble
{

/**
 * A filter for video frames which can apply lens projection changes
 * and/or video stabilisation
 * Abstract base class for filters. A filter is an object which consumes input
 * frames and produces output frames, optionally applying projection changes
 * and/or motion stabilisation.
 */
class FilterBase
{


@@ 36,6 37,9 @@ class FilterBase
    virtual cl_mem get_output_frame_buffer();

  public:
    /**
     * Construct a new filter with the given configuration
     */
    FilterBase(FilterConfig config);
    virtual ~FilterBase();
    /**

M include/filter_config.hpp => include/filter_config.hpp +110 -2
@@ 8,28 8,77 @@

namespace dewobble
{

/**
 * Pixel interpolation algorithm
 */
typedef enum Interpolation {
    /**
     * Nearest neighbour
     */
    DEWOBBLE_INTERPOLATION_NEAREST,
    /**
     * Bilinear
     */
    DEWOBBLE_INTERPOLATION_LINEAR,
    /**
     * Cubic
     */
    DEWOBBLE_INTERPOLATION_CUBIC,
    /**
     * Lanczos4 (over an 8x8 pixel neighbourhood)
     */
    DEWOBBLE_INTERPOLATION_LANCZOS4,
} Interpolation;

/**
 * Border extrapolation algorithm
 *
 * Used to color output pixels that do not map to the input image
 */
typedef enum BorderType {
    /**
     * Fill the border with a constant colour
     */
    DEWOBBLE_BORDER_CONSTANT,
    /**
     * Fill each unmapped pixel with the colour of the closest mapped pixel
     */
    DEWOBBLE_BORDER_REPLICATE,
    /**
     * Reflect the input image about the edges
     */
    DEWOBBLE_BORDER_REFLECT,
    /**
     * Wrap around to the opposite side of the source image
     */
    DEWOBBLE_BORDER_WRAP,
    /**
     * Reflect the input image about the center of the row/column of pixels on
     * the edge
     */
    DEWOBBLE_BORDER_REFLECT_101,
} BorderType;

/**
 * A constant colour
 */
class Color
{
  public:
    /**
     * OpenCV scalar value of the color in BGRA
     */
    cv::Scalar scalar;
    /**
     * @brief Construct a new Color object, with parameters in the range [0,1]
     */
    Color(double blue, double green, double red, double alpha);
};

/**
 * Configuration for a dewobble::Filter
 */
class FilterConfig
{
    Camera m_input_camera;


@@ 43,41 92,100 @@ class FilterConfig
    cl_device_id m_opencl_device;

  public:
    /**
     * Construct a new filter configuration
     *
     * @param input_camera Input camera which will be used to interpret the
     * input frames
     * @param output_camera Iutput camera which will be used to generate the
     * output frames
     * @param stabilizer An instance of dewobble::Stabilizer which will be used
     * to apply a corrective rotation to frames in order to smooth the camera
     * rotation over time
     */
    FilterConfig(
        Camera input_camera,
        Camera output_camera,
        std::shared_ptr<Stabilizer> stabilizer);

    /**
     * Get input camera which is used to interpret input frames
     */
    Camera get_input_camera() const;

    /**
     * Get output camera which is used to generate frames
     */
    Camera get_output_camera() const;

    /**
     * Get the stabilizer which is used to apply a corrective rotation to frames
     * in order to smooth the camera rotation over time
     */
    std::shared_ptr<Stabilizer> get_stabilizer() const;

    /**
     * Set the interpolation algorithm to use to map the output pixels to the
     * input (default: DEWOBBLE_INTERPOLATION_LINEAR)
     */
    void set_interpolation(Interpolation interpolation);
    /**
     * Get the interpolation algorithm to use to map the output pixels to the
     * input
     */
    Interpolation get_interpolation() const;

    /**
     * Set the border extrapolation algorithm used to colour pixels in
     * the output which do not map to the input image (default:
     * DEWOBBLE_BORDER_CONSTANT)
     */
    void set_border_type(BorderType border_type);
    /**
     * Get the border extrapolation algorithm used to colour pixels in
     * the output which do not map to the input image
     */
    BorderType get_border_type() const;

    /**
     * Set the color to fill the unmapped areas of the output image, in
     * the case that the border type is DEWOBBLE_BORDER_CONSTANT (default:
     * black)
     */
    void set_border_color(const Color border_color);
    /**
     * Get the color to fill the unmapped areas of the output image, in
     * the case that the border type is DEWOBBLE_BORDER_CONSTANT
     */
    Color get_border_color() const;

    /**
     * Set whether to include graphical debugging information in the output
     * images
     */
    void set_debug(bool debug);
    /**
     * @brief Get whether graphical debugging information is included in the
     * output images
     */
    bool get_debug() const;

    /**
     * Set the OpenCL context to be used for transforming frames
     * @param context the OpenCL context
     */
    void set_opencl_context(cl_context context);
    /**
     * Get the OpenCL context used for transforming frames
     */
    cl_context get_opencl_context();

    /**
     * Set the OpenCL device to be used for transforming frames
     * @param device the OpenCL device
     */
    void set_opencl_device(cl_device_id device);
    /**
     * Get the OpenCL device used for transforming frames
     */
    cl_device_id get_opencl_device();
};
} // namespace dewobble

M include/filter_sync.hpp => include/filter_sync.hpp +11 -0
@@ 14,6 14,14 @@

namespace dewobble
{

/**
 * A filter implentation that operates synchronously within the same thread.
 *
 * @note Using this filter will cause OpenCV's global per-thread OpenCL
 * context to be set for the thread where it is used, and the context also must
 * not be set elsewhere in that thread.
 */
class FilterSync : public FilterBase
{
    // The filter configuration


@@ 54,6 62,9 @@ class FilterSync : public FilterBase
    void init_after_changed_opencl_context();

  public:
    /**
     * @copydoc FilterBase::FilterBase()
     */
    FilterSync(FilterConfig config);

    void push_frame(cl_mem input_buffer, void *extra);

M include/filter_threaded.hpp => include/filter_threaded.hpp +8 -3
@@ 73,9 73,11 @@ typedef std::variant<
typedef std::variant<MessageOutputFrame, MessageAcknowledgeFrame> MessageOutput;

/**
 * A filter which works synchronously but delegates all processing to a
 * separate thread. This allows isolating the OpenCV thread-local OpenCL
 * execution context from the client thread.
 * A filter implementation which delegates all processing to a separate thread.
 * This allows isolating OpenCV's global per-thread execution context from the
 * calling thread. Compared to dewobble::FilterSync, one extra frame is kept in
 * flight to assist with keeping the worker thread busy. Otherwise, the
 * behaviour is the same as dewobble::FilterSync.
 */
class FilterThreaded : public FilterBase
{


@@ 92,6 94,9 @@ class FilterThreaded : public FilterBase

  public:
    ~FilterThreaded();
    /**
     * @copydoc FilterBase::FilterBase()
     */
    FilterThreaded(FilterConfig filter_config);

    void push_frame(cl_mem input_buffer, void *extra);

M include/safe_queue.hpp => include/safe_queue.hpp +5 -0
@@ 6,6 6,9 @@
#include <mutex>
#include <queue>

namespace dewobble
{

template <class T> class SafeQueue
{
    std::queue<std::unique_ptr<T>> m_queue;


@@ 37,4 40,6 @@ template <class T> class SafeQueue
    }
};

} // namespace dewobble

#endif // DEWOBBLE_SAFE_QUEUE_HPP
\ No newline at end of file

M include/stabilizer.hpp => include/stabilizer.hpp +38 -0
@@ 21,6 21,14 @@ class StabilizedFrame : public Frame
    FrameDebugInfo debug_info;
};

/**
 * Abstract base class for stabilizers
 *
 * A stabilizer is a class which consumes input frames and produces output
 * frames, while annotating the output frames with corrective rotations to
 * compensate for noisy camera rotation in the input.
 *
 */
class Stabilizer
{
  protected:


@@ 35,6 43,10 @@ class Stabilizer
    virtual StabilizedFrame pop_stabilized_frame();
};

/**
 * Stabilizer which uses a Savitzky-Golay filter on the detected input camera
 * rotations to produce a smooth output camera path
 */
class StabilizerSavitzkyGolay : public Stabilizer
{
    const unsigned int m_radius;


@@ 44,6 56,16 @@ class StabilizerSavitzkyGolay : public Stabilizer
    void try_pull_frames_from_detector();

  public:
    /**
     * @brief Construct a new Savitzky-Golay stabilizer
     *
     * @param camera the input camera
     * @param radius the number of frames to look ahead and behind the current
     * frame when deciding how to rotate the frame
     * @param interpolation_horizon maximum number of input frames to buffer to
     * assist with interpolating rotation on input frames where it could not be
     * detected
     */
    StabilizerSavitzkyGolay(
        Camera camera,
        unsigned int radius,


@@ 54,12 76,20 @@ class StabilizerSavitzkyGolay : public Stabilizer
    StabilizedFrame pop_stabilized_frame();
};

/**
 * Stabilizer which does not perform any stabilization, and simply passes
 * through frames.
 */
class StabilizerNone : public Stabilizer
{
  public:
    void push_frame(Frame frame);
};

/**
 * Stabilizer which maintains the position of the camera as it was in the first
 * input frame (aka tripod mode)
 */
class StabilizerFixed : public Stabilizer
{
    const Camera m_camera;


@@ 67,6 97,14 @@ class StabilizerFixed : public Stabilizer
    void flush_frames();

  public:
    /**
     * @brief Construct a new fixed stabilizer
     *
     * @param camera the input camera
     * @param interpolation_horizon maximum number of input frames to buffer to
     * assist with interpolating rotation on input frames where it could not be
     * detected
     */
    StabilizerFixed(Camera camera, int interpolation_horizon = 0);
    void push_frame(Frame frame);
    void end_input();