From e75fb87f861192ed0fac5d2758bcc752e85ca6e7 Mon Sep 17 00:00:00 2001 From: Doug Oucharek Date: Mon, 9 Dec 2013 22:56:54 +0800 Subject: [PATCH] staging/lustre/lnet: Add LNet Router Priority parameter This change adds a priority parameter to the route module settings. This paramter can be >= 0. Like hops, the lower the prioirty number the higher the priority. So lower numbered priorities will be selected over higher numbers. Lustre-change: http://review.whamcloud.com/5663 Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-2934 Signed-off-by: Doug Oucharek Reviewed-by: Andreas Dilger Reviewed-by: Amir Shehata Reviewed-by: Isaac Huang Reviewed-by: Oleg Drokin Signed-off-by: Peng Tao Signed-off-by: Andreas Dilger Signed-off-by: Greg Kroah-Hartman --- .../include/linux/libcfs/libcfs_ioctl.h | 1 + .../lustre/include/linux/lnet/lib-lnet.h | 5 ++- .../lustre/include/linux/lnet/lib-types.h | 2 +- drivers/staging/lustre/lnet/lnet/api-ni.c | 5 ++- drivers/staging/lustre/lnet/lnet/config.c | 39 ++++++++++++++++++- drivers/staging/lustre/lnet/lnet/lib-move.c | 6 +++ drivers/staging/lustre/lnet/lnet/router.c | 19 +++++---- .../staging/lustre/lnet/lnet/router_proc.c | 16 ++++---- 8 files changed, 72 insertions(+), 21 deletions(-) diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_ioctl.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_ioctl.h index dd1a1f1ef1f6..74dda57b98a8 100644 --- a/drivers/staging/lustre/include/linux/libcfs/libcfs_ioctl.h +++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_ioctl.h @@ -69,6 +69,7 @@ struct libcfs_ioctl_data { char ioc_bulk[0]; }; +#define ioc_priority ioc_u32[0] struct libcfs_ioctl_hdr { __u32 ioc_len; diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h index bf301048c7ab..3ac2bb5fd2db 100644 --- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h +++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h @@ -650,12 +650,13 @@ extern lnet_ni_t *lnet_net2ni(__u32 net); int lnet_notify(lnet_ni_t *ni, lnet_nid_t peer, int alive, cfs_time_t when); void lnet_notify_locked(lnet_peer_t *lp, int notifylnd, int alive, cfs_time_t when); -int lnet_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway_nid); +int lnet_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway_nid, + unsigned int priority); int lnet_check_routes(void); int lnet_del_route(__u32 net, lnet_nid_t gw_nid); void lnet_destroy_routes(void); int lnet_get_route(int idx, __u32 *net, __u32 *hops, - lnet_nid_t *gateway, __u32 *alive); + lnet_nid_t *gateway, __u32 *alive, __u32 *priority); void lnet_proc_init(void); void lnet_proc_fini(void); int lnet_rtrpools_alloc(int im_a_router); diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h index e579e7ed5070..dd8edcf1b5c0 100644 --- a/drivers/staging/lustre/include/linux/lnet/lib-types.h +++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h @@ -478,7 +478,6 @@ typedef struct lnet_peer { lnet_rc_data_t *lp_rcd; /* router checker state */ } lnet_peer_t; - /* peer hash size */ #define LNET_PEER_HASH_BITS 9 #define LNET_PEER_HASH_SIZE (1 << LNET_PEER_HASH_BITS) @@ -504,6 +503,7 @@ typedef struct { int lr_seq; /* sequence for round-robin */ unsigned int lr_downis; /* number of down NIs */ unsigned int lr_hops; /* how far I am */ + unsigned int lr_priority; /* route priority */ } lnet_route_t; #define LNET_REMOTE_NETS_HASH_DEFAULT (1U << 7) diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c index 5719959c8005..c562ff3e9283 100644 --- a/drivers/staging/lustre/lnet/lnet/api-ni.c +++ b/drivers/staging/lustre/lnet/lnet/api-ni.c @@ -1436,7 +1436,7 @@ LNetCtl(unsigned int cmd, void *arg) case IOC_LIBCFS_ADD_ROUTE: rc = lnet_add_route(data->ioc_net, data->ioc_count, - data->ioc_nid); + data->ioc_nid, data->ioc_priority); return (rc != 0) ? rc : lnet_check_routes(); case IOC_LIBCFS_DEL_ROUTE: @@ -1445,7 +1445,8 @@ LNetCtl(unsigned int cmd, void *arg) case IOC_LIBCFS_GET_ROUTE: return lnet_get_route(data->ioc_count, &data->ioc_net, &data->ioc_count, - &data->ioc_nid, &data->ioc_flags); + &data->ioc_nid, &data->ioc_flags, + &data->ioc_priority); case IOC_LIBCFS_NOTIFY_ROUTER: return lnet_notify(NULL, data->ioc_nid, data->ioc_flags, cfs_time_current() - diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c index de323f779db8..6a07b0a65d12 100644 --- a/drivers/staging/lustre/lnet/lnet/config.c +++ b/drivers/staging/lustre/lnet/lnet/config.c @@ -603,6 +603,37 @@ lnet_parse_hops(char *str, unsigned int *hops) *hops > 0 && *hops < 256); } +#define LNET_PRIORITY_SEPARATOR (':') + +int +lnet_parse_priority(char *str, unsigned int *priority, char **token) +{ + int nob; + char *sep; + int len; + + sep = strchr(str, LNET_PRIORITY_SEPARATOR); + if (sep == NULL) { + *priority = 0; + return 0; + } + len = strlen(sep + 1); + + if ((sscanf((sep+1), "%u%n", priority, &nob) < 1) || (len != nob)) { + /* Update the caller's token pointer so it treats the found + priority as the token to report in the error message. */ + *token += sep - str + 1; + return -1; + } + + CDEBUG(D_NET, "gateway %s, priority %d, nob %d\n", str, *priority, nob); + + /* + * Change priority separator to \0 to be able to parse NID + */ + *sep = '\0'; + return 0; +} int lnet_parse_route(char *str, int *im_a_router) @@ -624,6 +655,7 @@ lnet_parse_route(char *str, int *im_a_router) int myrc = -1; unsigned int hops; int got_hops = 0; + unsigned int priority = 0; INIT_LIST_HEAD(&gateways); INIT_LIST_HEAD(&nets); @@ -691,6 +723,11 @@ lnet_parse_route(char *str, int *im_a_router) LNET_NETTYP(net) == LOLND) goto token_error; } else { + rc = lnet_parse_priority(ltb->ltb_text, + &priority, &token); + if (rc < 0) + goto token_error; + nid = libcfs_str2nid(ltb->ltb_text); if (nid == LNET_NID_ANY || LNET_NETTYP(LNET_NIDNET(nid)) == LOLND) @@ -720,7 +757,7 @@ lnet_parse_route(char *str, int *im_a_router) continue; } - rc = lnet_add_route(net, hops, nid); + rc = lnet_add_route(net, hops, nid, priority); if (rc != 0) { CERROR("Can't create route to %s via %s\n", libcfs_net2str(net), diff --git a/drivers/staging/lustre/lnet/lnet/lib-move.c b/drivers/staging/lustre/lnet/lnet/lib-move.c index 2d611136da8f..bbf43ae04ed0 100644 --- a/drivers/staging/lustre/lnet/lnet/lib-move.c +++ b/drivers/staging/lustre/lnet/lnet/lib-move.c @@ -1074,6 +1074,12 @@ lnet_compare_routes(lnet_route_t *r1, lnet_route_t *r2) lnet_peer_t *p1 = r1->lr_gateway; lnet_peer_t *p2 = r2->lr_gateway; + if (r1->lr_priority < r2->lr_priority) + return 1; + + if (r1->lr_priority > r2->lr_priority) + return -1; + if (r1->lr_hops < r2->lr_hops) return 1; diff --git a/drivers/staging/lustre/lnet/lnet/router.c b/drivers/staging/lustre/lnet/lnet/router.c index 80b5ed1e9d64..d1ee44232eef 100644 --- a/drivers/staging/lustre/lnet/lnet/router.c +++ b/drivers/staging/lustre/lnet/lnet/router.c @@ -301,7 +301,8 @@ lnet_add_route_to_rnet (lnet_remotenet_t *rnet, lnet_route_t *route) } int -lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) +lnet_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway, + unsigned int priority) { struct list_head *e; lnet_remotenet_t *rnet; @@ -311,8 +312,8 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) int add_route; int rc; - CDEBUG(D_NET, "Add route: net %s hops %u gw %s\n", - libcfs_net2str(net), hops, libcfs_nid2str(gateway)); + CDEBUG(D_NET, "Add route: net %s hops %u priority %u gw %s\n", + libcfs_net2str(net), hops, priority, libcfs_nid2str(gateway)); if (gateway == LNET_NID_ANY || LNET_NETTYP(LNET_NIDNET(gateway)) == LOLND || @@ -342,6 +343,7 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) rnet->lrn_net = net; route->lr_hops = hops; route->lr_net = net; + route->lr_priority = priority; lnet_net_lock(LNET_LOCK_EX); @@ -552,7 +554,7 @@ lnet_destroy_routes (void) int lnet_get_route(int idx, __u32 *net, __u32 *hops, - lnet_nid_t *gateway, __u32 *alive) + lnet_nid_t *gateway, __u32 *alive, __u32 *priority) { struct list_head *e1; struct list_head *e2; @@ -574,10 +576,11 @@ lnet_get_route(int idx, __u32 *net, __u32 *hops, lr_list); if (idx-- == 0) { - *net = rnet->lrn_net; - *hops = route->lr_hops; - *gateway = route->lr_gateway->lp_nid; - *alive = route->lr_gateway->lp_alive; + *net = rnet->lrn_net; + *hops = route->lr_hops; + *priority = route->lr_priority; + *gateway = route->lr_gateway->lp_nid; + *alive = route->lr_gateway->lp_alive; lnet_net_unlock(cpt); return 0; } diff --git a/drivers/staging/lustre/lnet/lnet/router_proc.c b/drivers/staging/lustre/lnet/lnet/router_proc.c index e4fe7a116fb5..20d53e08705e 100644 --- a/drivers/staging/lustre/lnet/lnet/router_proc.c +++ b/drivers/staging/lustre/lnet/lnet/router_proc.c @@ -174,8 +174,8 @@ int LL_PROC_PROTO(proc_lnet_routes) the_lnet.ln_routing ? "enabled" : "disabled"); LASSERT(tmpstr + tmpsiz - s > 0); - s += snprintf(s, tmpstr + tmpsiz - s, "%-8s %4s %7s %s\n", - "net", "hops", "state", "router"); + s += snprintf(s, tmpstr + tmpsiz - s, "%-8s %4s %8s %7s %s\n", + "net", "hops", "priority", "state", "router"); LASSERT(tmpstr + tmpsiz - s > 0); lnet_net_lock(0); @@ -229,14 +229,16 @@ int LL_PROC_PROTO(proc_lnet_routes) } if (route != NULL) { - __u32 net = rnet->lrn_net; - unsigned int hops = route->lr_hops; - lnet_nid_t nid = route->lr_gateway->lp_nid; - int alive = route->lr_gateway->lp_alive; + __u32 net = rnet->lrn_net; + unsigned int hops = route->lr_hops; + unsigned int priority = route->lr_priority; + lnet_nid_t nid = route->lr_gateway->lp_nid; + int alive = route->lr_gateway->lp_alive; s += snprintf(s, tmpstr + tmpsiz - s, - "%-8s %4u %7s %s\n", + "%-8s %4u %8u %7s %s\n", libcfs_net2str(net), hops, + priority, alive ? "up" : "down", libcfs_nid2str(nid)); LASSERT(tmpstr + tmpsiz - s > 0); -- 2.30.2