libceph: fix mutex coverage for ceph_con_close

Hold the mutex while twiddling all of the state bits to avoid possible
races.  While we're here, make not of why we cannot close the socket
directly.

Signed-off-by: Sage Weil <sage@inktank.com>
Reviewed-by: Alex Elder <elder@inktank.com>
Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>

diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index 6e2f67816f61d4ccfa3aa580737b69ccc649e1a2..e65b15d5d8b92ad4dd6841e459aeff8fe10ba325 100644 (file)
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -503,6 +503,7 @@ static void reset_connection(struct ceph_connection *con)
  */
 void ceph_con_close(struct ceph_connection *con)
 {
+       mutex_lock(&con->mutex);
        dout("con_close %p peer %s\n", con,
             ceph_pr_addr(&con->peer_addr.in_addr));
        clear_bit(NEGOTIATING, &con->state);
@@ -515,11 +516,16 @@ void ceph_con_close(struct ceph_connection *con)
        clear_bit(KEEPALIVE_PENDING, &con->flags);
        clear_bit(WRITE_PENDING, &con->flags);
 
-       mutex_lock(&con->mutex);
        reset_connection(con);
        con->peer_global_seq = 0;
        cancel_delayed_work(&con->work);
        mutex_unlock(&con->mutex);
+
+       /*
+        * We cannot close the socket directly from here because the
+        * work threads use it without holding the mutex.  Instead, let
+        * con_work() do it.
+        */
        queue_con(con);
 }
 EXPORT_SYMBOL(ceph_con_close);