Received-SPF: pass (google.com: domain of linux-bluetooth+bounces-3282-linux.lists.archive=gmail.com@vger.kernel.org designates 2604:1380:45d1:ec00::1 as permitted sender) client-ip=2604:1380:45d1:ec00::1;
Precedence: bulk
MIME-Version: 1.0
References: <20240405093834.63666-1-surban@surban.net> <20240405102551.114999-1-surban@surban.net>
In-Reply-To: <20240405102551.114999-1-surban@surban.net>
From: Luiz Augusto von Dentz <luiz.dentz@gmail.com>
Date: Fri, 5 Apr 2024 11:30:49 -0400
Message-ID: <CABBYNZJfbgDoHr=ZBqf3Rckz7=34Lyq4RRyKbNfpi2qSO6PfqA@mail.gmail.com>
Subject: Re: [PATCH v2] Bluetooth: keep LE flow credits when recvbuf full
To: Sebastian Urban <surban@surban.net>
Cc: Marcel Holtmann <marcel@holtmann.org>, Johan Hedberg <johan.hedberg@gmail.com>, 
	"David S. Miller" <davem@davemloft.net>, Eric Dumazet <edumazet@google.com>, 
	Jakub Kicinski <kuba@kernel.org>, Paolo Abeni <pabeni@redhat.com>, linux-bluetooth@vger.kernel.org, 
	netdev@vger.kernel.org, linux-kernel@vger.kernel.org
Content-Type: text/plain; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable

Hi Sebastian,

On Fri, Apr 5, 2024 at 6:26=E2=80=AFAM Sebastian Urban <surban@surban.net> =
wrote:
>
> Previously LE flow credits were returned to the
> sender even if the socket's receive buffer was
> full. This meant that no back-pressure
> was applied to the sender, thus it continued to
> send data, resulting in data loss without any
> error being reported.
>
> This is fixed by stopping the return of LE flow
> credits when the receive buffer of an L2CAP socket
> is full. Returning of the credits is resumed, once
> the receive buffer is half-empty.
>
> Already received data is temporary stored within
> l2cap_pinfo, since Bluetooth LE provides no
> retransmission mechanism once the data has been
> acked by the physical layer.
>
> Signed-off-by: Sebastian Urban <surban@surban.net>
> ---
>  include/net/bluetooth/l2cap.h |  7 ++++-
>  net/bluetooth/l2cap_core.c    | 38 ++++++++++++++++++++++---
>  net/bluetooth/l2cap_sock.c    | 53 ++++++++++++++++++++++++++---------
>  3 files changed, 79 insertions(+), 19 deletions(-)
>
> diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.=
h
> index 92d7197f9a56..230c14ea944c 100644
> --- a/include/net/bluetooth/l2cap.h
> +++ b/include/net/bluetooth/l2cap.h
> @@ -682,10 +682,15 @@ struct l2cap_user {
>  /* ----- L2CAP socket info ----- */
>  #define l2cap_pi(sk) ((struct l2cap_pinfo *) sk)
>
> +struct l2cap_rx_busy {
> +       struct list_head        list;
> +       struct sk_buff          *skb;
> +};

In theory we only really to queue 1 skb at most, since we would stop
giving credits, or perhaps this is because we had given enough credits
for MTU + 1, so the +1 segment could result in a second SDU/skb to be
completed while waiting the user space process to start reading again?

>  struct l2cap_pinfo {
>         struct bt_sock          bt;
>         struct l2cap_chan       *chan;
> -       struct sk_buff          *rx_busy_skb;
> +       struct list_head        rx_busy;
>  };
>
>  enum {
> diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c
> index ab5a9d42fae7..c78af7fad255 100644
> --- a/net/bluetooth/l2cap_core.c
> +++ b/net/bluetooth/l2cap_core.c
> @@ -63,6 +63,8 @@ static void l2cap_retrans_timeout(struct work_struct *w=
ork);
>  static void l2cap_monitor_timeout(struct work_struct *work);
>  static void l2cap_ack_timeout(struct work_struct *work);
>
> +static void l2cap_chan_le_send_credits(struct l2cap_chan *chan);

We probably need to change the way send_credits calculates the number
of credits to be restored, it needs to consider the actual available
buffer size at the socket rather then assuming we always shall have
space for MTU + 1, that way the remote side would always have the
exact information of how much buffer space is left. That said perhaps
we need a way to inform when user space reads then we need to call
into send_credits again.

>  static inline u8 bdaddr_type(u8 link_type, u8 bdaddr_type)
>  {
>         if (link_type =3D=3D LE_LINK) {
> @@ -5714,17 +5716,34 @@ static int l2cap_resegment(struct l2cap_chan *cha=
n)
>         return 0;
>  }
>
> -void l2cap_chan_busy(struct l2cap_chan *chan, int busy)
> +static void l2cap_chan_busy_ertm(struct l2cap_chan *chan, int busy)
>  {
>         u8 event;
>
> -       if (chan->mode !=3D L2CAP_MODE_ERTM)
> -               return;
> -
>         event =3D busy ? L2CAP_EV_LOCAL_BUSY_DETECTED : L2CAP_EV_LOCAL_BU=
SY_CLEAR;
>         l2cap_tx(chan, NULL, NULL, event);
>  }
>
> +static void l2cap_chan_busy_le(struct l2cap_chan *chan, int busy)
> +{
> +       if (busy) {
> +               set_bit(CONN_LOCAL_BUSY, &chan->conn_state);
> +       } else {
> +               clear_bit(CONN_LOCAL_BUSY, &chan->conn_state);
> +               l2cap_chan_le_send_credits(chan);
> +       }
> +}
> +
> +void l2cap_chan_busy(struct l2cap_chan *chan, int busy)
> +{
> +       if (chan->mode =3D=3D L2CAP_MODE_ERTM) {
> +               l2cap_chan_busy_ertm(chan, busy);
> +       } else if (chan->mode =3D=3D L2CAP_MODE_LE_FLOWCTL ||
> +                  chan->mode =3D=3D L2CAP_MODE_EXT_FLOWCTL) {
> +               l2cap_chan_busy_le(chan, busy);
> +       }
> +}
> +
>  static int l2cap_rx_queued_iframes(struct l2cap_chan *chan)
>  {
>         int err =3D 0;
> @@ -6514,6 +6533,11 @@ static void l2cap_chan_le_send_credits(struct l2ca=
p_chan *chan)
>         struct l2cap_le_credits pkt;
>         u16 return_credits;
>
> +       if (test_bit(CONN_LOCAL_BUSY, &chan->conn_state)) {
> +               BT_DBG("busy chan %p not returning credits to sender", ch=
an);
> +               return;
> +       }
> +
>         return_credits =3D (chan->imtu / chan->mps) + 1;
>
>         if (chan->rx_credits >=3D return_credits)
> @@ -6542,6 +6566,12 @@ static int l2cap_ecred_recv(struct l2cap_chan *cha=
n, struct sk_buff *skb)
>         /* Wait recv to confirm reception before updating the credits */
>         err =3D chan->ops->recv(chan, skb);
>
> +       if (err < 0) {
> +               BT_ERR("Queueing received LE L2CAP data failed");
> +               l2cap_send_disconn_req(chan, ECONNRESET);
> +               return err;
> +       }
> +
>         /* Update credits whenever an SDU is received */
>         l2cap_chan_le_send_credits(chan);
>
> diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c
> index ee7a41d6994f..3b0fb6e0b61b 100644
> --- a/net/bluetooth/l2cap_sock.c
> +++ b/net/bluetooth/l2cap_sock.c
> @@ -1177,7 +1177,9 @@ static int l2cap_sock_recvmsg(struct socket *sock, =
struct msghdr *msg,
>         else
>                 err =3D bt_sock_recvmsg(sock, msg, len, flags);
>
> -       if (pi->chan->mode !=3D L2CAP_MODE_ERTM)
> +       if (pi->chan->mode !=3D L2CAP_MODE_ERTM &&
> +           pi->chan->mode !=3D L2CAP_MODE_LE_FLOWCTL &&
> +           pi->chan->mode !=3D L2CAP_MODE_EXT_FLOWCTL)
>                 return err;
>
>         /* Attempt to put pending rx data in the socket buffer */
> @@ -1187,11 +1189,15 @@ static int l2cap_sock_recvmsg(struct socket *sock=
, struct msghdr *msg,
>         if (!test_bit(CONN_LOCAL_BUSY, &pi->chan->conn_state))
>                 goto done;
>
> -       if (pi->rx_busy_skb) {
> -               if (!__sock_queue_rcv_skb(sk, pi->rx_busy_skb))
> -                       pi->rx_busy_skb =3D NULL;
> -               else
> +       while (!list_empty(&pi->rx_busy)) {
> +               struct l2cap_rx_busy *rx_busy =3D
> +                       list_first_entry(&pi->rx_busy,
> +                                        struct l2cap_rx_busy,
> +                                        list);
> +               if (__sock_queue_rcv_skb(sk, rx_busy->skb) < 0)
>                         goto done;
> +               list_del(&rx_busy->list);
> +               kfree(rx_busy);

I see now, this is trying to dequeue packets if the socket is read,
which in case we turn the send_credits function to calculate the
credits based on the socket buffer size that would not be necessary
but then we would need to call into send_credits here.

>         }
>
>         /* Restore data flow when half of the receive buffer is
> @@ -1459,17 +1465,20 @@ static struct l2cap_chan *l2cap_sock_new_connecti=
on_cb(struct l2cap_chan *chan)
>  static int l2cap_sock_recv_cb(struct l2cap_chan *chan, struct sk_buff *s=
kb)
>  {
>         struct sock *sk =3D chan->data;
> +       struct l2cap_pinfo *pi =3D l2cap_pi(sk);
>         int err;
>
>         lock_sock(sk);
>
> -       if (l2cap_pi(sk)->rx_busy_skb) {
> +       if (chan->mode =3D=3D L2CAP_MODE_ERTM && !list_empty(&pi->rx_busy=
)) {
>                 err =3D -ENOMEM;
>                 goto done;
>         }
>
>         if (chan->mode !=3D L2CAP_MODE_ERTM &&
> -           chan->mode !=3D L2CAP_MODE_STREAMING) {
> +           chan->mode !=3D L2CAP_MODE_STREAMING &&
> +           chan->mode !=3D L2CAP_MODE_LE_FLOWCTL &&
> +           chan->mode !=3D L2CAP_MODE_EXT_FLOWCTL) {
>                 /* Even if no filter is attached, we could potentially
>                  * get errors from security modules, etc.
>                  */
> @@ -1480,17 +1489,28 @@ static int l2cap_sock_recv_cb(struct l2cap_chan *=
chan, struct sk_buff *skb)
>
>         err =3D __sock_queue_rcv_skb(sk, skb);
>
> -       /* For ERTM, handle one skb that doesn't fit into the recv
> +       /* For ERTM and LE, handle a skb that doesn't fit into the recv
>          * buffer.  This is important to do because the data frames
>          * have already been acked, so the skb cannot be discarded.
>          *
>          * Notify the l2cap core that the buffer is full, so the
>          * LOCAL_BUSY state is entered and no more frames are
>          * acked and reassembled until there is buffer space
> -        * available.
> +        * available. In the case of LE this blocks returning of flow
> +        * credits.
>          */
> -       if (err < 0 && chan->mode =3D=3D L2CAP_MODE_ERTM) {
> -               l2cap_pi(sk)->rx_busy_skb =3D skb;
> +       if (err < 0 &&
> +           (chan->mode =3D=3D L2CAP_MODE_ERTM ||
> +            chan->mode =3D=3D L2CAP_MODE_LE_FLOWCTL ||
> +            chan->mode =3D=3D L2CAP_MODE_EXT_FLOWCTL)) {
> +               struct l2cap_rx_busy *rx_busy =3D
> +                       kmalloc(sizeof(*rx_busy), GFP_KERNEL);
> +               if (!rx_busy) {
> +                       err =3D -ENOMEM;
> +                       goto done;
> +               }
> +               rx_busy->skb =3D skb;
> +               list_add_tail(&rx_busy->list, &pi->rx_busy);
>                 l2cap_chan_busy(chan, 1);
>                 err =3D 0;
>         }
> @@ -1716,6 +1736,8 @@ static const struct l2cap_ops l2cap_chan_ops =3D {
>
>  static void l2cap_sock_destruct(struct sock *sk)
>  {
> +       struct l2cap_rx_busy *rx_busy, *next;
> +
>         BT_DBG("sk %p", sk);
>
>         if (l2cap_pi(sk)->chan) {
> @@ -1723,9 +1745,10 @@ static void l2cap_sock_destruct(struct sock *sk)
>                 l2cap_chan_put(l2cap_pi(sk)->chan);
>         }
>
> -       if (l2cap_pi(sk)->rx_busy_skb) {
> -               kfree_skb(l2cap_pi(sk)->rx_busy_skb);
> -               l2cap_pi(sk)->rx_busy_skb =3D NULL;
> +       list_for_each_entry_safe(rx_busy, next, &l2cap_pi(sk)->rx_busy, l=
ist) {
> +               kfree_skb(rx_busy->skb);
> +               list_del(&rx_busy->list);
> +               kfree(rx_busy);
>         }
>
>         skb_queue_purge(&sk->sk_receive_queue);
> @@ -1830,6 +1853,8 @@ static struct sock *l2cap_sock_alloc(struct net *ne=
t, struct socket *sock,
>         sk->sk_destruct =3D l2cap_sock_destruct;
>         sk->sk_sndtimeo =3D L2CAP_CONN_TIMEOUT;
>
> +       INIT_LIST_HEAD(&l2cap_pi(sk)->rx_busy);
> +
>         chan =3D l2cap_chan_create();
>         if (!chan) {
>                 sk_free(sk);
> --
> 2.34.1
>


--=20
Luiz Augusto von Dentz