staging: lustre: Dynamic LNet Configuration (DLC)

This is the first patch of a set of patches that enables DLC.

This patch adds some cleanup in the config.c as well as some
preparatory changes in peer.c to enable dynamic network
configuration

Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-2456
Change-Id: I8c8bbf3b55acf4d76f22a8be587b553a70d31889
Reviewed-on: http://review.whamcloud.com/9830
Reviewed-by: Liang Zhen <liang.zhen@intel.com>
Reviewed-by: James Simmons <uja.ornl@gmail.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
index b2e50173593128871cbcd1b7e6487caf755f4795..77d8e37c7acaa5cd8c89388bf5d05198d7aee312 100644 (file)
--- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
@@ -685,7 +685,7 @@ int lnet_parse_networks(struct list_head *nilist, char *networks);
 int lnet_nid2peer_locked(lnet_peer_t **lpp, lnet_nid_t nid, int cpt);
 lnet_peer_t *lnet_find_peer_locked(struct lnet_peer_table *ptable,
                                   lnet_nid_t nid);
-void lnet_peer_tables_cleanup(void);
+void lnet_peer_tables_cleanup(lnet_ni_t *ni);
 void lnet_peer_tables_destroy(void);
 int lnet_peer_tables_create(void);
 void lnet_debug_peer(lnet_nid_t nid);

diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h
index d769c356723c65bd3a9c184c7a45305c7ef29328..be650d4ba5f29a4c611b3a328d52166eba99cc01 100644 (file)
--- a/drivers/staging/lustre/include/linux/lnet/lib-types.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h
@@ -351,6 +351,8 @@ typedef struct lnet_peer {
 struct lnet_peer_table {
        int                      pt_version;    /* /proc validity stamp */
        int                      pt_number;     /* # peers extant */
+       /* # zombies to go to deathrow (and not there yet) */
+       int                      pt_zombies;
        struct list_head         pt_deathrow;   /* zombie peers */
        struct list_head        *pt_hash;       /* NID->peer hash */
 };
@@ -616,9 +618,6 @@ typedef struct {
        /* registered LNDs */
        struct list_head                  ln_lnds;
 
-       /* space for network names */
-       char                             *ln_network_tokens;
-       int                               ln_network_tokens_nob;
        /* test protocol compatibility flags */
        int                               ln_testprotocompat;
 

diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
index 58b30f1d4ac1dcb3ce87473b648619c386df088c..cd68ca73e309893efe1c91a9ee766a8d63b6a834 100644 (file)
--- a/drivers/staging/lustre/lnet/lnet/api-ni.c
+++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
@@ -892,7 +892,7 @@ lnet_shutdown_lndnis(void)
         * Clear the peer table and wait for all peers to go (they hold refs on
         * their NIs)
         */
-       lnet_peer_tables_cleanup();
+       lnet_peer_tables_cleanup(NULL);
 
        lnet_net_lock(LNET_LOCK_EX);
        /*
@@ -952,12 +952,6 @@ lnet_shutdown_lndnis(void)
 
        the_lnet.ln_shutdown = 0;
        lnet_net_unlock(LNET_LOCK_EX);
-
-       if (the_lnet.ln_network_tokens) {
-               LIBCFS_FREE(the_lnet.ln_network_tokens,
-                           the_lnet.ln_network_tokens_nob);
-               the_lnet.ln_network_tokens = NULL;
-       }
 }
 
 static int

diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
index e817eb3a91e97999f37d17706e2fb7c4bcabea55..bcc97b422be108137786619e72d93c16180c59da 100644 (file)
--- a/drivers/staging/lustre/lnet/lnet/config.c
+++ b/drivers/staging/lustre/lnet/lnet/config.c
@@ -96,6 +96,8 @@ lnet_net_unique(__u32 net, struct list_head *nilist)
 void
 lnet_ni_free(struct lnet_ni *ni)
 {
+       int i;
+
        if (ni->ni_refs)
                cfs_percpt_free(ni->ni_refs);
 
@@ -105,6 +107,10 @@ lnet_ni_free(struct lnet_ni *ni)
        if (ni->ni_cpts)
                cfs_expr_list_values_free(ni->ni_cpts, ni->ni_ncpts);
 
+       for (i = 0; i < LNET_MAX_INTERFACES && ni->ni_interfaces[i]; i++) {
+               LIBCFS_FREE(ni->ni_interfaces[i],
+                           strlen(ni->ni_interfaces[i]) + 1);
+       }
        LIBCFS_FREE(ni, sizeof(*ni));
 }
 
@@ -199,8 +205,6 @@ lnet_parse_networks(struct list_head *nilist, char *networks)
                return -ENOMEM;
        }
 
-       the_lnet.ln_network_tokens = tokens;
-       the_lnet.ln_network_tokens_nob = tokensize;
        memcpy(tokens, networks, tokensize);
        tmp = tokens;
        str = tokens;
@@ -321,7 +325,23 @@ lnet_parse_networks(struct list_head *nilist, char *networks)
                                goto failed;
                        }
 
-                       ni->ni_interfaces[niface++] = iface;
+                       /*
+                        * Allocate a separate piece of memory and copy
+                        * into it the string, so we don't have
+                        * a depencency on the tokens string.  This way we
+                        * can free the tokens at the end of the function.
+                        * The newly allocated ni_interfaces[] can be
+                        * freed when freeing the NI
+                        */
+                       LIBCFS_ALLOC(ni->ni_interfaces[niface],
+                                    strlen(iface) + 1);
+                       if (!ni->ni_interfaces[niface]) {
+                               CERROR("Can't allocate net interface name\n");
+                               goto failed;
+                       }
+                       strncpy(ni->ni_interfaces[niface], iface,
+                               strlen(iface));
+                       niface++;
                        iface = comma;
                } while (iface);
 
@@ -346,6 +366,8 @@ lnet_parse_networks(struct list_head *nilist, char *networks)
        }
 
        LASSERT(!list_empty(nilist));
+
+       LIBCFS_FREE(tokens, tokensize);
        return 0;
 
  failed_syntax:
@@ -362,7 +384,6 @@ lnet_parse_networks(struct list_head *nilist, char *networks)
                cfs_expr_list_free(el);
 
        LIBCFS_FREE(tokens, tokensize);
-       the_lnet.ln_network_tokens = NULL;
 
        return -EINVAL;
 }

diff --git a/drivers/staging/lustre/lnet/lnet/peer.c b/drivers/staging/lustre/lnet/lnet/peer.c
index 00086ee90dd8eed827d1ea005e467ca5a5a5f16b..42b2d44666d61000a74190071ef6c21db0475221 100644 (file)
--- a/drivers/staging/lustre/lnet/lnet/peer.c
+++ b/drivers/staging/lustre/lnet/lnet/peer.c
@@ -103,62 +103,116 @@ lnet_peer_tables_destroy(void)
        the_lnet.ln_peer_tables = NULL;
 }
 
+static void
+lnet_peer_table_cleanup_locked(lnet_ni_t *ni, struct lnet_peer_table *ptable)
+{
+       int i;
+       lnet_peer_t *lp;
+       lnet_peer_t *tmp;
+
+       for (i = 0; i < LNET_PEER_HASH_SIZE; i++) {
+               list_for_each_entry_safe(lp, tmp, &ptable->pt_hash[i],
+                                        lp_hashlist) {
+                       if (ni && ni != lp->lp_ni)
+                               continue;
+                       list_del_init(&lp->lp_hashlist);
+                       /* Lose hash table's ref */
+                       ptable->pt_zombies++;
+                       lnet_peer_decref_locked(lp);
+               }
+       }
+}
+
+static void
+lnet_peer_table_deathrow_wait_locked(struct lnet_peer_table *ptable,
+                                    int cpt_locked)
+{
+       int i;
+
+       for (i = 3; ptable->pt_zombies; i++) {
+               lnet_net_unlock(cpt_locked);
+
+               if (is_power_of_2(i)) {
+                       CDEBUG(D_WARNING,
+                              "Waiting for %d zombies on peer table\n",
+                              ptable->pt_zombies);
+               }
+               set_current_state(TASK_UNINTERRUPTIBLE);
+               schedule_timeout(cfs_time_seconds(1) >> 1);
+               lnet_net_lock(cpt_locked);
+       }
+}
+
+static void
+lnet_peer_table_del_rtrs_locked(lnet_ni_t *ni, struct lnet_peer_table *ptable,
+                               int cpt_locked)
+{
+       lnet_peer_t *lp;
+       lnet_peer_t *tmp;
+       lnet_nid_t lp_nid;
+       int i;
+
+       for (i = 0; i < LNET_PEER_HASH_SIZE; i++) {
+               list_for_each_entry_safe(lp, tmp, &ptable->pt_hash[i],
+                                        lp_hashlist) {
+                       if (ni != lp->lp_ni)
+                               continue;
+
+                       if (!lp->lp_rtr_refcount)
+                               continue;
+
+                       lp_nid = lp->lp_nid;
+
+                       lnet_net_unlock(cpt_locked);
+                       lnet_del_route(LNET_NIDNET(LNET_NID_ANY), lp_nid);
+                       lnet_net_lock(cpt_locked);
+               }
+       }
+}
+
 void
-lnet_peer_tables_cleanup(void)
+lnet_peer_tables_cleanup(lnet_ni_t *ni)
 {
        struct lnet_peer_table *ptable;
+       struct list_head deathrow;
+       lnet_peer_t *lp;
        int i;
-       int j;
 
-       LASSERT(the_lnet.ln_shutdown);  /* i.e. no new peers */
+       INIT_LIST_HEAD(&deathrow);
 
+       LASSERT(the_lnet.ln_shutdown || ni);
+       /*
+        * If just deleting the peers for a NI, get rid of any routes these
+        * peers are gateways for.
+        */
        cfs_percpt_for_each(ptable, i, the_lnet.ln_peer_tables) {
                lnet_net_lock(i);
-
-               for (j = 0; j < LNET_PEER_HASH_SIZE; j++) {
-                       struct list_head *peers = &ptable->pt_hash[j];
-
-                       while (!list_empty(peers)) {
-                               lnet_peer_t *lp = list_entry(peers->next,
-                                                                lnet_peer_t,
-                                                                lp_hashlist);
-                               list_del_init(&lp->lp_hashlist);
-                               /* lose hash table's ref */
-                               lnet_peer_decref_locked(lp);
-                       }
-               }
-
+               lnet_peer_table_del_rtrs_locked(ni, ptable, i);
                lnet_net_unlock(i);
        }
 
+       /*
+        * Start the process of moving the applicable peers to
+        * deathrow.
+        */
        cfs_percpt_for_each(ptable, i, the_lnet.ln_peer_tables) {
-               LIST_HEAD(deathrow);
-               lnet_peer_t *lp;
-
                lnet_net_lock(i);
+               lnet_peer_table_cleanup_locked(ni, ptable);
+               lnet_net_unlock(i);
+       }
 
-               for (j = 3; ptable->pt_number; j++) {
-                       lnet_net_unlock(i);
-
-                       if (!(j & (j - 1))) {
-                               CDEBUG(D_WARNING,
-                                      "Waiting for %d peers on peer table\n",
-                                      ptable->pt_number);
-                       }
-                       set_current_state(TASK_UNINTERRUPTIBLE);
-                       schedule_timeout(cfs_time_seconds(1) / 2);
-                       lnet_net_lock(i);
-               }
+       /* Cleanup all entries on deathrow. */
+       cfs_percpt_for_each(ptable, i, the_lnet.ln_peer_tables) {
+               lnet_net_lock(i);
+               lnet_peer_table_deathrow_wait_locked(ptable, i);
                list_splice_init(&ptable->pt_deathrow, &deathrow);
-
                lnet_net_unlock(i);
+       }
 
-               while (!list_empty(&deathrow)) {
-                       lp = list_entry(deathrow.next,
-                                       lnet_peer_t, lp_hashlist);
-                       list_del(&lp->lp_hashlist);
-                       LIBCFS_FREE(lp, sizeof(*lp));
-               }
+       while (!list_empty(&deathrow)) {
+               lp = list_entry(deathrow.next, lnet_peer_t, lp_hashlist);
+               list_del(&lp->lp_hashlist);
+               LIBCFS_FREE(lp, sizeof(*lp));
        }
 }
 
@@ -181,6 +235,8 @@ lnet_destroy_peer_locked(lnet_peer_t *lp)
        lp->lp_ni = NULL;
 
        list_add(&lp->lp_hashlist, &ptable->pt_deathrow);
+       LASSERT(ptable->pt_zombies > 0);
+       ptable->pt_zombies--;
 }
 
 lnet_peer_t *