diff --git a/filters/f_enhancement_pair.c b/filters/f_enhancement_pair.c
index 2c11a3a3de..7b74d10fc1 100644
--- a/filters/f_enhancement_pair.c
+++ b/filters/f_enhancement_pair.c
@@ -29,7 +29,7 @@
 #define PTS_MATCH_TOLERANCE 1e-6
 
 // Number of frames hold for matching.
-#define QUEUE_MAX 16
+#define QUEUE_MAX MP_ENHANCEMENT_PAIR_QUEUE_MAX
 
 struct priv {
     struct mp_decoder_wrapper *el_dec;
@@ -226,6 +226,8 @@ struct mp_filter *mp_enhancement_pair_create(struct mp_filter *parent,
 
     struct priv *p = f->priv;
     p->el_dec = mp_decoder_wrapper_create(f, el_sh);
+    if (p->el_dec)
+        mp_decoder_wrapper_set_extra_hw_frames(p->el_dec, QUEUE_MAX);
     if (!p->el_dec || !mp_decoder_wrapper_reinit(p->el_dec)) {
         MP_WARN(f, "Failed to set up enhancement-layer decoder; "
                 "rendering base layer only.\n");
diff --git a/filters/f_enhancement_pair.h b/filters/f_enhancement_pair.h
index dc00e7ca14..cfe1fb8b96 100644
--- a/filters/f_enhancement_pair.h
+++ b/filters/f_enhancement_pair.h
@@ -21,6 +21,9 @@
 
 struct sh_stream;
 
+// Maximum number of decoded frames the pairing filter retains while matching.
+#define MP_ENHANCEMENT_PAIR_QUEUE_MAX 8
+
 // Enhancement-layer pairing filter.
 //
 // Reads base-layer (BL) mp_image frames from its input pin, decodes the
diff --git a/player/video.c b/player/video.c
index e2cc4801f2..eabd544018 100644
--- a/player/video.c
+++ b/player/video.c
@@ -39,6 +39,7 @@
 #include "sub/osd.h"
 #include "video/hwdec.h"
 #include "filters/f_decoder_wrapper.h"
+#include "filters/f_enhancement_pair.h"
 #include "video/out/vo.h"
 
 #include "core.h"
@@ -192,6 +193,9 @@ int init_video_decoder(struct MPContext *mpctx, struct track *track)
     if (!track->dec)
         goto err_out;
 
+    if (sh_stream_dependent_sibling(track->stream))
+        mp_decoder_wrapper_set_extra_hw_frames(track->dec, MP_ENHANCEMENT_PAIR_QUEUE_MAX);
+
     if (!mp_decoder_wrapper_reinit(track->dec))
         goto err_out;