multimedia/ffmpeg: fix DoS in VC-2 encoder

Reported by:	Vladimir Krstulja
Obtained from:	upstream (release/3.4 branch)
Security:	CVE-2017-16840

1 files changed, 83 insertions, 0 deletions

diff --git a/multimedia/ffmpeg/files/patch-CVE-2017-16840 b/multimedia/ffmpeg/files/patch-CVE-2017-16840
new file mode 100644
index 000000000000..58be7b24fa16
--- /dev/null
+++ b/multimedia/ffmpeg/files/patch-CVE-2017-16840
@@ -0,0 +1,83 @@
+commit a94cb36ab2ad99d3a1331c9f91831ef593d94f74
+Author: Rostislav Pehlivanov <atomnuker@gmail.com>
+Date:   Wed Nov 8 23:50:04 2017 +0000
+
+    vc2enc_dwt: pad the temporary buffer by the slice size
+    
+    Since non-Haar wavelets need to look into pixels outside the frame, we
+    need to pad the buffer. The old factor of two seemed to be a workaround
+    that fact and only padded to the left and bottom. This correctly pads
+    by the slice size and as such reduces memory usage and potential
+    exploits.
+    Reported by Liu Bingchang.
+    
+    Ideally, there should be no temporary buffer but the encoder is designed
+    to deinterleave the coefficients into the classical wavelet structure
+    with the lower frequency values in the top left corner.
+    
+    Signed-off-by: Rostislav Pehlivanov <atomnuker@gmail.com>
+    (cherry picked from commit 3228ac730c11eca49d5680d5550128e397061c85)
+
+--- libavcodec/vc2enc.c.orig	2017-10-15 15:59:37 UTC
++++ libavcodec/vc2enc.c
+@@ -1190,7 +1190,8 @@ static av_cold int vc2_encode_init(AVCodecContext *avc
+         /* DWT init */
+         if (ff_vc2enc_init_transforms(&s->transform_args[i].t,
+                                       s->plane[i].coef_stride,
+-                                      s->plane[i].dwt_height))
++                                      s->plane[i].dwt_height,
++                                      s->slice_width, s->slice_height))
+             goto alloc_fail;
+     }
+ 
+--- libavcodec/vc2enc_dwt.c.orig	2017-09-12 00:51:34 UTC
++++ libavcodec/vc2enc_dwt.c
+@@ -255,21 +255,27 @@ static void vc2_subband_dwt_haar_shift(VC2TransformCon
+     dwt_haar(t, data, stride, width, height, 1);
+ }
+ 
+-av_cold int ff_vc2enc_init_transforms(VC2TransformContext *s, int p_width, int p_height)
++av_cold int ff_vc2enc_init_transforms(VC2TransformContext *s, int p_stride,
++                                      int p_height, int slice_w, int slice_h)
+ {
+     s->vc2_subband_dwt[VC2_TRANSFORM_9_7]    = vc2_subband_dwt_97;
+     s->vc2_subband_dwt[VC2_TRANSFORM_5_3]    = vc2_subband_dwt_53;
+     s->vc2_subband_dwt[VC2_TRANSFORM_HAAR]   = vc2_subband_dwt_haar;
+     s->vc2_subband_dwt[VC2_TRANSFORM_HAAR_S] = vc2_subband_dwt_haar_shift;
+ 
+-    s->buffer = av_malloc(2*p_width*p_height*sizeof(dwtcoef));
++    /* Pad by the slice size, only matters for non-Haar wavelets */
++    s->buffer = av_calloc((p_stride + slice_w)*(p_height + slice_h), sizeof(dwtcoef));
+     if (!s->buffer)
+         return 1;
+ 
++    s->padding = (slice_h >> 1)*p_stride + (slice_w >> 1);
++    s->buffer += s->padding;
++
+     return 0;
+ }
+ 
+ av_cold void ff_vc2enc_free_transforms(VC2TransformContext *s)
+ {
+-    av_freep(&s->buffer);
++    av_free(s->buffer - s->padding);
++    s->buffer = NULL;
+ }
+--- libavcodec/vc2enc_dwt.h.orig	2017-09-12 00:51:34 UTC
++++ libavcodec/vc2enc_dwt.h
+@@ -41,12 +41,14 @@ enum VC2TransformType {
+ 
+ typedef struct VC2TransformContext {
+     dwtcoef *buffer;
++    int padding;
+     void (*vc2_subband_dwt[VC2_TRANSFORMS_NB])(struct VC2TransformContext *t,
+                                                dwtcoef *data, ptrdiff_t stride,
+                                                int width, int height);
+ } VC2TransformContext;
+ 
+-int  ff_vc2enc_init_transforms(VC2TransformContext *t, int p_width, int p_height);
++int  ff_vc2enc_init_transforms(VC2TransformContext *t, int p_stride, int p_height,
++                               int slice_w, int slice_h);
+ void ff_vc2enc_free_transforms(VC2TransformContext *t);
+ 
+ #endif /* AVCODEC_VC2ENC_DWT_H */