ALSA: pcm: oss: Fix negative period/buffer sizes

The period size calculation in OSS layer may receive a negative value
as an error, but the code there assumes only the positive values and
handle them with size_t.  Due to that, a too big value may be passed
to the lower layers.

This patch changes the code to handle with ssize_t and adds the proper
error checks appropriately.

Reported-by: syzbot+bb348e9f9a954d42746f@syzkaller.appspotmail.com
Reported-by: Bixuan Cui <cuibixuan@linux.alibaba.com>
Cc: <stable@vger.kernel.org>
Link: https://lore.kernel.org/r/1638270978-42412-1-git-send-email-cuibixuan@linux.alibaba.com
Link: https://lore.kernel.org/r/20211201073606.11660-2-tiwai@suse.de
Signed-off-by: Takashi Iwai <tiwai@suse.de>

diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c
index 82a8187..bec7590 100644 (file)
--- a/sound/core/oss/pcm_oss.c
+++ b/sound/core/oss/pcm_oss.c
@@ -147,7 +147,7 @@ snd_pcm_hw_param_value_min(const struct snd_pcm_hw_params *params,
  *
  * Return the maximum value for field PAR.
  */
-static unsigned int
+static int
 snd_pcm_hw_param_value_max(const struct snd_pcm_hw_params *params,
                           snd_pcm_hw_param_t var, int *dir)
 {
@@ -682,18 +682,24 @@ static int snd_pcm_oss_period_size(struct snd_pcm_substream *substream,
                                   struct snd_pcm_hw_params *oss_params,
                                   struct snd_pcm_hw_params *slave_params)
 {
-       size_t s;
-       size_t oss_buffer_size, oss_period_size, oss_periods;
-       size_t min_period_size, max_period_size;
+       ssize_t s;
+       ssize_t oss_buffer_size;
+       ssize_t oss_period_size, oss_periods;
+       ssize_t min_period_size, max_period_size;
        struct snd_pcm_runtime *runtime = substream->runtime;
        size_t oss_frame_size;
 
        oss_frame_size = snd_pcm_format_physical_width(params_format(oss_params)) *
                         params_channels(oss_params) / 8;
 
+       oss_buffer_size = snd_pcm_hw_param_value_max(slave_params,
+                                                    SNDRV_PCM_HW_PARAM_BUFFER_SIZE,
+                                                    NULL);
+       if (oss_buffer_size <= 0)
+               return -EINVAL;
        oss_buffer_size = snd_pcm_plug_client_size(substream,
-                                                  snd_pcm_hw_param_value_max(slave_params, SNDRV_PCM_HW_PARAM_BUFFER_SIZE, NULL)) * oss_frame_size;
-       if (!oss_buffer_size)
+                                                  oss_buffer_size * oss_frame_size);
+       if (oss_buffer_size <= 0)
                return -EINVAL;
        oss_buffer_size = rounddown_pow_of_two(oss_buffer_size);
        if (atomic_read(&substream->mmap_count)) {
@@ -730,7 +736,7 @@ static int snd_pcm_oss_period_size(struct snd_pcm_substream *substream,
 
        min_period_size = snd_pcm_plug_client_size(substream,
                                                   snd_pcm_hw_param_value_min(slave_params, SNDRV_PCM_HW_PARAM_PERIOD_SIZE, NULL));
-       if (min_period_size) {
+       if (min_period_size > 0) {
                min_period_size *= oss_frame_size;
                min_period_size = roundup_pow_of_two(min_period_size);
                if (oss_period_size < min_period_size)
@@ -739,7 +745,7 @@ static int snd_pcm_oss_period_size(struct snd_pcm_substream *substream,
 
        max_period_size = snd_pcm_plug_client_size(substream,
                                                   snd_pcm_hw_param_value_max(slave_params, SNDRV_PCM_HW_PARAM_PERIOD_SIZE, NULL));
-       if (max_period_size) {
+       if (max_period_size > 0) {
                max_period_size *= oss_frame_size;
                max_period_size = rounddown_pow_of_two(max_period_size);
                if (oss_period_size > max_period_size)
@@ -752,7 +758,7 @@ static int snd_pcm_oss_period_size(struct snd_pcm_substream *substream,
                oss_periods = substream->oss.setup.periods;
 
        s = snd_pcm_hw_param_value_max(slave_params, SNDRV_PCM_HW_PARAM_PERIODS, NULL);
-       if (runtime->oss.maxfrags && s > runtime->oss.maxfrags)
+       if (s > 0 && runtime->oss.maxfrags && s > runtime->oss.maxfrags)
                s = runtime->oss.maxfrags;
        if (oss_periods > s)
                oss_periods = s;