client API, dxva2: add a workaround for OpenGL fullscreen issues

This is basically a hack for drivers which prevent the mpv DXVA2 decoder
glue from working if OpenGL is in fullscreen mode.

Since it doesn't add any "hard" new API to the client API, some of the
code would be required for a true zero-copy hw decoding pipeline, and
sine it isn't too much code after all, this is probably acceptable.

DOCS/client-api-changes.rst

@@ -32,6 +32,8 @@ API changes
 
 ::
 
+ ...    - add "GL_MP_D3D_interfaces" pseudo extension to make it possible to
+          use DXVA2 in OpenGL fullscreen mode in some situations
  1.19   - mpv_request_log_messages() now accepts "terminal-default" as parameter
  1.18   - add MPV_END_FILE_REASON_REDIRECT, and change behavior of
           MPV_EVENT_END_FILE accordingly

libmpv/opengl_cb.h

@@ -107,6 +107,24 @@ extern "C" {
  * When the mpv core is destroyed (e.g. via mpv_terminate_destroy()), the OpenGL
  * context must have been uninitialized. If this doesn't happen, undefined
  * behavior will result.
+ *
+ * Special D3D interop considerations
+ * ----------------------------------
+ *
+ * If OpenGL switches to fullscreen, most players give it access GPU access,
+ * which means DXVA2 hardware decoding in mpv won't work. This can be worked
+ * around by giving mpv access to Direct3D device, which it will then use to
+ * create a decoder. The device can be either the real device used for display,
+ * or a "blank" device created before switching to fullscreen.
+ *
+ * You can do this by adding "GL_MP_D3D_interfaces" to the additional extension
+ * string when calling mpv_opengl_cb_init_gl(). The get_proc_address callback
+ * should resolve a function named "glMPGetD3DInterface", which has the
+ * signature: "void* __stdcall glMPGetD3DInterface(const char* name)". If
+ * name is "IDirect3DDevice9", it should return a IDirect3DDevice9 pointer
+ * (or NULL if not available). libmpv will release this interface when it is
+ * done with it (usually when mpv_opengl_cb_uninit_gl() is called). New
+ * interface names can be added in the future.
  */
 
 /**

video/decode/dxva2.c

@@ -675,6 +675,7 @@ static int dxva2_init_decoder(struct lavc_ctx *s, int fmt, int w, int h)
 static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                  const char *decoder)
 {
+    hwdec_request_api(info, "dxva2"); // we can do without too
     for (int i = 0; dxva2_modes[i].guid; i++) {
         const dxva2_mode *mode = &dxva2_modes[i];
         if (mp_codec_to_av_codec_id(decoder) == mode->codec)

video/out/gl_common.c

@@ -306,6 +306,15 @@ static const struct gl_functions gl_functions[] = {
             {0}
         },
     },
+    // These don't exist - they are for the sake of mpv internals, and libmpv
+    // interaction (see libmpv/opengl_cb.h).
+    {
+        .extension = "GL_MP_D3D_interfaces",
+        .functions = (const struct gl_function[]) {
+            DEF_FN(MPGetD3DInterface),
+            {0}
+        },
+    },
 };
 
 #undef FN_OFFS

video/out/gl_common.h

@@ -275,6 +275,8 @@ struct GL {
 
     void (GLAPIENTRY *DebugMessageCallback)(MP_GLDEBUGPROC callback,
                                             const void *userParam);
+
+    void *(GLAPIENTRY *MPGetD3DInterface)(const char *name);
 };
 
 #endif /* MPLAYER_GL_COMMON_H */

video/out/gl_hwdec.c

@@ -32,6 +32,7 @@
 extern const struct gl_hwdec_driver gl_hwdec_vaglx;
 extern const struct gl_hwdec_driver gl_hwdec_vda;
 extern const struct gl_hwdec_driver gl_hwdec_vdpau;
+extern const struct gl_hwdec_driver gl_hwdec_dxva2;
 
 static const struct gl_hwdec_driver *const mpgl_hwdec_drivers[] = {
 #if HAVE_VAAPI_GLX
@@ -42,6 +43,9 @@ static const struct gl_hwdec_driver *const mpgl_hwdec_drivers[] = {
 #endif
 #if HAVE_VDA_GL
     &gl_hwdec_vda,
+#endif
+#if HAVE_DXVA2_HWACCEL
+    &gl_hwdec_dxva2,
 #endif
     NULL
 };

video/out/gl_hwdec_dxva2.c

@@ -0,0 +1,64 @@
+#include "common/common.h"
+
+#include "gl_hwdec.h"
+#include "gl_utils.h"
+#include "video/d3d.h"
+#include "video/hwdec.h"
+
+// This does not provide real (zero-copy) interop - it merely exists for
+// making sure the same D3D device is used for decoding and display, which
+// may help with OpenGL fullscreen mode.
+
+struct priv {
+    struct mp_d3d_ctx ctx;
+};
+
+static void destroy(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+    if (p->ctx.d3d9_device)
+        IDirect3DDevice9_Release(p->ctx.d3d9_device);
+}
+
+static int create(struct gl_hwdec *hw)
+{
+    GL *gl = hw->gl;
+    if (hw->hwctx || !gl->MPGetD3DInterface)
+        return -1;
+
+    struct priv *p = talloc_zero(hw, struct priv);
+    hw->priv = p;
+
+    p->ctx.d3d9_device = gl->MPGetD3DInterface("IDirect3DDevice9");
+    if (!p->ctx.d3d9_device)
+        return -1;
+
+    p->ctx.hwctx.type = HWDEC_DXVA2_COPY;
+    p->ctx.hwctx.d3d_ctx = &p->ctx;
+
+    MP_VERBOSE(hw, "Using libmpv supplied device %p.\n", p->ctx.d3d9_device);
+
+    hw->hwctx = &p->ctx.hwctx;
+    hw->converted_imgfmt = 0;
+    return 0;
+}
+
+static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
+{
+    return -1;
+}
+
+static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     GLuint *out_textures)
+{
+    return -1;
+}
+
+const struct gl_hwdec_driver gl_hwdec_dxva2 = {
+    .api_name = "dxva2",
+    .imgfmt = -1,
+    .create = create,
+    .reinit = reinit,
+    .map_image = map_image,
+    .destroy = destroy,
+};

wscript_build.py

@@ -334,6 +334,7 @@ def build(ctx):
         ( "video/out/gl_common.c",               "gl" ),
         ( "video/out/gl_rpi.c",                  "rpi-gles" ),
         ( "video/out/gl_hwdec.c",                "gl" ),
+        ( "video/out/gl_hwdec_dxva2.c",          "gl-win32" ),
         ( "video/out/gl_hwdec_vaglx.c",          "vaapi-glx" ),
         ( "video/out/gl_hwdec_vda.c",            "vda-gl" ),
         ( "video/out/gl_hwdec_vdpau.c",          "vdpau-gl-x11" ),