net: Rearrange dst_entry layout to avoid useless padding.

We have padding to try and align the refcount on a separate cache
line.  But after several simplifications the padding has increased
substantially.

So now it's easy to change the layout to get rid of the padding
entirely.

We group the write-heavy __refcnt and __use with less often used
items such as the rcu_head and the error code.

Signed-off-by: David S. Miller <davem@davemloft.net>
Reviewed-by: Eric Dumazet <edumazet@google.com>

diff --git a/include/net/dst.h b/include/net/dst.h
index 424bff66f16d781b74bf480cea6c9aa96df80943..2270513d0790730178946335de78d9d54ebe1a7e 100644 (file)
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -34,7 +34,6 @@ struct sk_buff;
 
 struct dst_entry {
        struct net_device       *dev;
-       struct rcu_head         rcu_head;
        struct  dst_ops         *ops;
        unsigned long           _metrics;
        unsigned long           expires;
@@ -56,8 +55,6 @@ struct dst_entry {
 #define DST_XFRM_QUEUE         0x0040
 #define DST_METADATA           0x0080
 
-       short                   error;
-
        /* A non-zero value of dst->obsolete forces by-hand validation
         * of the route entry.  Positive values are set by the generic
         * dst layer to indicate that the entry has been forcefully
@@ -73,29 +70,25 @@ struct dst_entry {
 #define DST_OBSOLETE_KILL      -2
        unsigned short          header_len;     /* more space at head required */
        unsigned short          trailer_len;    /* space to reserve at tail */
-       unsigned short          __pad3;
-
-#ifdef CONFIG_IP_ROUTE_CLASSID
-       __u32                   tclassid;
-#else
-       __u32                   __pad2;
-#endif
 
-#ifdef CONFIG_64BIT
-       /*
-        * Align __refcnt to a 64 bytes alignment
-        * (L1_CACHE_SIZE would be too much)
-        */
-       long                    __pad_to_align_refcnt[5];
-#endif
        /*
         * __refcnt wants to be on a different cache line from
         * input/output/ops or performance tanks badly
         */
-       atomic_t                __refcnt;       /* client references    */
+#ifdef CONFIG_64BIT
+       atomic_t                __refcnt;       /* 64-bit offset 64 */
+#endif
        int                     __use;
        unsigned long           lastuse;
        struct lwtunnel_state   *lwtstate;
+       struct rcu_head         rcu_head;
+       short                   error;
+       short                   __pad;
+       __u32                   tclassid;
+#ifndef CONFIG_64BIT
+       atomic_t                __refcnt;       /* 32-bit offset 64 */
+#endif
+
        union {
                struct dst_entry        *next;
        };
@@ -244,7 +237,7 @@ static inline void dst_hold(struct dst_entry *dst)
 {
        /*
         * If your kernel compilation stops here, please check
-        * __pad_to_align_refcnt declaration in struct dst_entry
+        * the placement of __refcnt in struct dst_entry
         */
        BUILD_BUG_ON(offsetof(struct dst_entry, __refcnt) & 63);
        WARN_ON(atomic_inc_not_zero(&dst->__refcnt) == 0);