staging: rtl8712: check for alloc fail in _r8712_init_recv_priv()

The function _r8712_init_recv_priv() and also r8712_init_recv_priv()
just returns silently if they fail to allocate memory. Change their
return type to int and add necessary checks and handling if they return
-ENOMEM

Tested-by: Philipp Hortmann <philipp.g.hortmann@gmail.com>
Signed-off-by: Nam Cao <namcaov@gmail.com>
Link: https://lore.kernel.org/r/506ac35a667e511db568b06b86834fd0ceeba453.1666688642.git.namcaov@gmail.com
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/drivers/staging/rtl8712/os_intfs.c b/drivers/staging/rtl8712/os_intfs.c
index 003e972..12adb47 100644 (file)
--- a/drivers/staging/rtl8712/os_intfs.c
+++ b/drivers/staging/rtl8712/os_intfs.c
@@ -309,7 +309,9 @@ int r8712_init_drv_sw(struct _adapter *padapter)
        if (ret)
                return ret;
        _r8712_init_xmit_priv(&padapter->xmitpriv, padapter);
-       _r8712_init_recv_priv(&padapter->recvpriv, padapter);
+       ret = _r8712_init_recv_priv(&padapter->recvpriv, padapter);
+       if (ret)
+               return ret;
        memset((unsigned char *)&padapter->securitypriv, 0,
               sizeof(struct security_priv));
        timer_setup(&padapter->securitypriv.tkip_timer,

diff --git a/drivers/staging/rtl8712/recv_osdep.h b/drivers/staging/rtl8712/recv_osdep.h
index d8c1fa7..fbe3f28 100644 (file)
--- a/drivers/staging/rtl8712/recv_osdep.h
+++ b/drivers/staging/rtl8712/recv_osdep.h
@@ -18,15 +18,15 @@
 #include "drv_types.h"
 #include <linux/skbuff.h>
 
-void _r8712_init_recv_priv(struct recv_priv *precvpriv,
-                          struct _adapter *padapter);
+int _r8712_init_recv_priv(struct recv_priv *precvpriv,
+                         struct _adapter *padapter);
 void _r8712_free_recv_priv(struct recv_priv *precvpriv);
 void r8712_recv_entry(union recv_frame *precv_frame);
 void r8712_recv_indicatepkt(struct _adapter *adapter,
                            union recv_frame *precv_frame);
 void r8712_handle_tkip_mic_err(struct _adapter *padapter, u8 bgroup);
-void r8712_init_recv_priv(struct recv_priv *precvpriv,
-                         struct _adapter *padapter);
+int r8712_init_recv_priv(struct recv_priv *precvpriv,
+                        struct _adapter *padapter);
 void r8712_free_recv_priv(struct recv_priv *precvpriv);
 void r8712_os_recv_resource_alloc(struct _adapter *padapter,
                                  union recv_frame *precvframe);

diff --git a/drivers/staging/rtl8712/rtl8712_recv.c b/drivers/staging/rtl8712/rtl8712_recv.c
index 7f1fdd0..7da014a 100644 (file)
--- a/drivers/staging/rtl8712/rtl8712_recv.c
+++ b/drivers/staging/rtl8712/rtl8712_recv.c
@@ -30,8 +30,8 @@
 
 static void recv_tasklet(struct tasklet_struct *t);
 
-void r8712_init_recv_priv(struct recv_priv *precvpriv,
-                         struct _adapter *padapter)
+int r8712_init_recv_priv(struct recv_priv *precvpriv,
+                        struct _adapter *padapter)
 {
        int i;
        struct recv_buf *precvbuf;
@@ -44,7 +44,7 @@ void r8712_init_recv_priv(struct recv_priv *precvpriv,
        precvpriv->pallocated_recv_buf =
                kzalloc(NR_RECVBUFF * sizeof(struct recv_buf) + 4, GFP_ATOMIC);
        if (!precvpriv->pallocated_recv_buf)
-               return;
+               return -ENOMEM;
        precvpriv->precv_buf = precvpriv->pallocated_recv_buf + 4 -
                              ((addr_t)(precvpriv->pallocated_recv_buf) & 3);
        precvbuf = (struct recv_buf *)precvpriv->precv_buf;
@@ -75,6 +75,7 @@ void r8712_init_recv_priv(struct recv_priv *precvpriv,
                }
                pskb = NULL;
        }
+       return 0;
 }
 
 void r8712_free_recv_priv(struct recv_priv *precvpriv)

diff --git a/drivers/staging/rtl8712/rtl871x_recv.c b/drivers/staging/rtl8712/rtl871x_recv.c
index 4db7eed..8a35662 100644 (file)
--- a/drivers/staging/rtl8712/rtl871x_recv.c
+++ b/drivers/staging/rtl8712/rtl871x_recv.c
@@ -42,9 +42,10 @@ void _r8712_init_sta_recv_priv(struct sta_recv_priv *psta_recvpriv)
        _init_queue(&psta_recvpriv->defrag_q);
 }
 
-void _r8712_init_recv_priv(struct recv_priv *precvpriv,
-                          struct _adapter *padapter)
+int _r8712_init_recv_priv(struct recv_priv *precvpriv,
+                         struct _adapter *padapter)
 {
+       int ret;
        sint i;
        union recv_frame *precvframe;
 
@@ -58,7 +59,7 @@ void _r8712_init_recv_priv(struct recv_priv *precvpriv,
                                sizeof(union recv_frame) + RXFRAME_ALIGN_SZ,
                                GFP_ATOMIC);
        if (!precvpriv->pallocated_frame_buf)
-               return;
+               return -ENOMEM;
        precvpriv->precv_frame_buf = precvpriv->pallocated_frame_buf +
                                    RXFRAME_ALIGN_SZ -
                                    ((addr_t)(precvpriv->pallocated_frame_buf) &
@@ -73,7 +74,11 @@ void _r8712_init_recv_priv(struct recv_priv *precvpriv,
                precvframe++;
        }
        precvpriv->rx_pending_cnt = 1;
-       r8712_init_recv_priv(precvpriv, padapter);
+       ret = r8712_init_recv_priv(precvpriv, padapter);
+       if (ret)
+               kfree(precvpriv->pallocated_frame_buf);
+
+       return ret;
 }
 
 void _r8712_free_recv_priv(struct recv_priv *precvpriv)