git.openwrt.org Git - openwrt/staging/blogic.git/blob

1 /*

2 * This file is part of the Chelsio T4 Ethernet driver for Linux.

3 *

5 *

6 * This software is available to you under a choice of one of two

7 * licenses. You may choose to be licensed under the terms of the GNU

8 * General Public License (GPL) Version 2, available from the file

9 * COPYING in the main directory of this source tree, or the

10 * OpenIB.org BSD license below:

11 *

12 * Redistribution and use in source and binary forms, with or

13 * without modification, are permitted provided that the following

14 * conditions are met:

15 *

16 * - Redistributions of source code must retain the above

17 * copyright notice, this list of conditions and the following

18 * disclaimer.

19 *

20 * - Redistributions in binary form must reproduce the above

21 * copyright notice, this list of conditions and the following

22 * disclaimer in the documentation and/or other materials

23 * provided with the distribution.

24 *

25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS

29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN

30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN

31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

32 * SOFTWARE.

33 */

35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

37 #include <linux/bitmap.h>

38 #include <linux/crc32.h>

39 #include <linux/ctype.h>

40 #include <linux/debugfs.h>

41 #include <linux/err.h>

42 #include <linux/etherdevice.h>

43 #include <linux/firmware.h>

44 #include <linux/if.h>

45 #include <linux/if_vlan.h>

46 #include <linux/init.h>

47 #include <linux/log2.h>

48 #include <linux/mdio.h>

49 #include <linux/module.h>

50 #include <linux/moduleparam.h>

51 #include <linux/mutex.h>

52 #include <linux/netdevice.h>

53 #include <linux/pci.h>

54 #include <linux/aer.h>

55 #include <linux/rtnetlink.h>

56 #include <linux/sched.h>

57 #include <linux/seq_file.h>

58 #include <linux/sockios.h>

59 #include <linux/vmalloc.h>

60 #include <linux/workqueue.h>

61 #include <net/neighbour.h>

62 #include <net/netevent.h>

63 #include <net/addrconf.h>

64 #include <asm/uaccess.h>

66 #include "cxgb4.h"

67 #include "t4_regs.h"

68 #include "t4_msg.h"

69 #include "t4fw_api.h"

70 #include "l2t.h"

72 #include <../drivers/net/bonding/bonding.h>

74 #ifdef DRV_VERSION

75 #undef DRV_VERSION

76 #endif

77 #define DRV_VERSION "2.0.0-ko"

78 #define DRV_DESC "Chelsio T4/T5 Network Driver"

80 /*

81 * Max interrupt hold-off timer value in us. Queues fall back to this value

82 * under extreme memory pressure so it's largish to give the system time to

83 * recover.

84 */

85 #define MAX_SGE_TIMERVAL 200U

87 enum {

88 /*

89 * Physical Function provisioning constants.

90 */

91 PFRES_NVI = 4, /* # of Virtual Interfaces */

92 PFRES_NETHCTRL = 128, /* # of EQs used for ETH or CTRL Qs */

93 PFRES_NIQFLINT = 128, /* # of ingress Qs/w Free List(s)/intr

94 */

95 PFRES_NEQ = 256, /* # of egress queues */

96 PFRES_NIQ = 0, /* # of ingress queues */

97 PFRES_TC = 0, /* PCI-E traffic class */

98 PFRES_NEXACTF = 128, /* # of exact MPS filters */

100 PFRES_R_CAPS = FW_CMD_CAP_PF,

101 PFRES_WX_CAPS = FW_CMD_CAP_PF,

102

103 #ifdef CONFIG_PCI_IOV

104 /*

105 * Virtual Function provisioning constants. We need two extra Ingress

106 * Queues with Interrupt capability to serve as the VF's Firmware

107 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --

108 * neither will have Free Lists associated with them). For each

109 * Ethernet/Control Egress Queue and for each Free List, we need an

110 * Egress Context.

111 */

112 VFRES_NPORTS = 1, /* # of "ports" per VF */

113 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */

114

115 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */

116 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */

117 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */

118 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */

119 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */

120 VFRES_TC = 0, /* PCI-E traffic class */

121 VFRES_NEXACTF = 16, /* # of exact MPS filters */

122

123 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,

124 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,

125 #endif

126 };

127

128 /*

129 * Provide a Port Access Rights Mask for the specified PF/VF. This is very

130 * static and likely not to be useful in the long run. We really need to

131 * implement some form of persistent configuration which the firmware

132 * controls.

133 */

134 static unsigned int pfvfres_pmask(struct adapter *adapter,

135 unsigned int pf, unsigned int vf)

136 {

137 unsigned int portn, portvec;

138

139 /*

140 * Give PF's access to all of the ports.

141 */

142 if (vf == 0)

143 return FW_PFVF_CMD_PMASK_MASK;

144

145 /*

146 * For VFs, we'll assign them access to the ports based purely on the

147 * PF. We assign active ports in order, wrapping around if there are

148 * fewer active ports than PFs: e.g. active port[pf % nports].

149 * Unfortunately the adapter's port_info structs haven't been

150 * initialized yet so we have to compute this.

151 */

152 if (adapter->params.nports == 0)

153 return 0;

154

155 portn = pf % adapter->params.nports;

156 portvec = adapter->params.portvec;

157 for (;;) {

158 /*

159 * Isolate the lowest set bit in the port vector. If we're at

160 * the port number that we want, return that as the pmask.

161 * otherwise mask that bit out of the port vector and

162 * decrement our port number ...

163 */

164 unsigned int pmask = portvec ^ (portvec & (portvec-1));

165 if (portn == 0)

166 return pmask;

167 portn--;

168 portvec &= ~pmask;

169 }

170 /*NOTREACHED*/

171 }

172

173 enum {

174 MAX_TXQ_ENTRIES = 16384,

175 MAX_CTRL_TXQ_ENTRIES = 1024,

176 MAX_RSPQ_ENTRIES = 16384,

177 MAX_RX_BUFFERS = 16384,

178 MIN_TXQ_ENTRIES = 32,

179 MIN_CTRL_TXQ_ENTRIES = 32,

180 MIN_RSPQ_ENTRIES = 128,

181 MIN_FL_ENTRIES = 16

182 };

183

184 /* Host shadow copy of ingress filter entry. This is in host native format

185 * and doesn't match the ordering or bit order, etc. of the hardware of the

186 * firmware command. The use of bit-field structure elements is purely to

187 * remind ourselves of the field size limitations and save memory in the case

188 * where the filter table is large.

189 */

190 struct filter_entry {

191 /* Administrative fields for filter.

192 */

193 u32 valid:1; /* filter allocated and valid */

194 u32 locked:1; /* filter is administratively locked */

195

196 u32 pending:1; /* filter action is pending firmware reply */

197 u32 smtidx:8; /* Source MAC Table index for smac */

198 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */

199

200 /* The filter itself. Most of this is a straight copy of information

201 * provided by the extended ioctl(). Some fields are translated to

202 * internal forms -- for instance the Ingress Queue ID passed in from

203 * the ioctl() is translated into the Absolute Ingress Queue ID.

204 */

205 struct ch_filter_specification fs;

206 };

207

208 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \

209 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\

210 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)

211

212 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }

213

214 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {

215 CH_DEVICE(0xa000, 0), /* PE10K */

216 CH_DEVICE(0x4001, -1),

217 CH_DEVICE(0x4002, -1),

218 CH_DEVICE(0x4003, -1),

219 CH_DEVICE(0x4004, -1),

220 CH_DEVICE(0x4005, -1),

221 CH_DEVICE(0x4006, -1),

222 CH_DEVICE(0x4007, -1),

223 CH_DEVICE(0x4008, -1),

224 CH_DEVICE(0x4009, -1),

225 CH_DEVICE(0x400a, -1),

226 CH_DEVICE(0x4401, 4),

227 CH_DEVICE(0x4402, 4),

228 CH_DEVICE(0x4403, 4),

229 CH_DEVICE(0x4404, 4),

230 CH_DEVICE(0x4405, 4),

231 CH_DEVICE(0x4406, 4),

232 CH_DEVICE(0x4407, 4),

233 CH_DEVICE(0x4408, 4),

234 CH_DEVICE(0x4409, 4),

235 CH_DEVICE(0x440a, 4),

236 CH_DEVICE(0x440d, 4),

237 CH_DEVICE(0x440e, 4),

238 CH_DEVICE(0x5001, 4),

239 CH_DEVICE(0x5002, 4),

240 CH_DEVICE(0x5003, 4),

241 CH_DEVICE(0x5004, 4),

242 CH_DEVICE(0x5005, 4),

243 CH_DEVICE(0x5006, 4),

244 CH_DEVICE(0x5007, 4),

245 CH_DEVICE(0x5008, 4),

246 CH_DEVICE(0x5009, 4),

247 CH_DEVICE(0x500A, 4),

248 CH_DEVICE(0x500B, 4),

249 CH_DEVICE(0x500C, 4),

250 CH_DEVICE(0x500D, 4),

251 CH_DEVICE(0x500E, 4),

252 CH_DEVICE(0x500F, 4),

253 CH_DEVICE(0x5010, 4),

254 CH_DEVICE(0x5011, 4),

255 CH_DEVICE(0x5012, 4),

256 CH_DEVICE(0x5013, 4),

257 CH_DEVICE(0x5014, 4),

258 CH_DEVICE(0x5015, 4),

259 CH_DEVICE(0x5401, 4),

260 CH_DEVICE(0x5402, 4),

261 CH_DEVICE(0x5403, 4),

262 CH_DEVICE(0x5404, 4),

263 CH_DEVICE(0x5405, 4),

264 CH_DEVICE(0x5406, 4),

265 CH_DEVICE(0x5407, 4),

266 CH_DEVICE(0x5408, 4),

267 CH_DEVICE(0x5409, 4),

268 CH_DEVICE(0x540A, 4),

269 CH_DEVICE(0x540B, 4),

270 CH_DEVICE(0x540C, 4),

271 CH_DEVICE(0x540D, 4),

272 CH_DEVICE(0x540E, 4),

273 CH_DEVICE(0x540F, 4),

274 CH_DEVICE(0x5410, 4),

275 CH_DEVICE(0x5411, 4),

276 CH_DEVICE(0x5412, 4),

277 CH_DEVICE(0x5413, 4),

278 CH_DEVICE(0x5414, 4),

279 CH_DEVICE(0x5415, 4),

280 { 0, }

281 };

282

283 #define FW4_FNAME "cxgb4/t4fw.bin"

284 #define FW5_FNAME "cxgb4/t5fw.bin"

285 #define FW4_CFNAME "cxgb4/t4-config.txt"

286 #define FW5_CFNAME "cxgb4/t5-config.txt"

287

288 MODULE_DESCRIPTION(DRV_DESC);

289 MODULE_AUTHOR("Chelsio Communications");

290 MODULE_LICENSE("Dual BSD/GPL");

291 MODULE_VERSION(DRV_VERSION);

292 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);

293 MODULE_FIRMWARE(FW4_FNAME);

294 MODULE_FIRMWARE(FW5_FNAME);

295

296 /*

297 * Normally we're willing to become the firmware's Master PF but will be happy

298 * if another PF has already become the Master and initialized the adapter.

299 * Setting "force_init" will cause this driver to forcibly establish itself as

300 * the Master PF and initialize the adapter.

301 */

302 static uint force_init;

303

304 module_param(force_init, uint, 0644);

305 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");

306

307 /*

308 * Normally if the firmware we connect to has Configuration File support, we

309 * use that and only fall back to the old Driver-based initialization if the

310 * Configuration File fails for some reason. If force_old_init is set, then

311 * we'll always use the old Driver-based initialization sequence.

312 */

313 static uint force_old_init;

314

315 module_param(force_old_init, uint, 0644);

316 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");

317

318 static int dflt_msg_enable = DFLT_MSG_ENABLE;

319

320 module_param(dflt_msg_enable, int, 0644);

321 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");

322

323 /*

324 * The driver uses the best interrupt scheme available on a platform in the

325 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which

326 * of these schemes the driver may consider as follows:

327 *

328 * msi = 2: choose from among all three options

329 * msi = 1: only consider MSI and INTx interrupts

330 * msi = 0: force INTx interrupts

331 */

332 static int msi = 2;

333

334 module_param(msi, int, 0644);

335 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");

336

337 /*

338 * Queue interrupt hold-off timer values. Queues default to the first of these

339 * upon creation.

340 */

341 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };

342

343 module_param_array(intr_holdoff, uint, NULL, 0644);

344 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "

345 "0..4 in microseconds");

346

347 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };

348

349 module_param_array(intr_cnt, uint, NULL, 0644);

350 MODULE_PARM_DESC(intr_cnt,

351 "thresholds 1..3 for queue interrupt packet counters");

352

353 /*

354 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers

355 * offset by 2 bytes in order to have the IP headers line up on 4-byte

356 * boundaries. This is a requirement for many architectures which will throw

357 * a machine check fault if an attempt is made to access one of the 4-byte IP

358 * header fields on a non-4-byte boundary. And it's a major performance issue

359 * even on some architectures which allow it like some implementations of the

360 * x86 ISA. However, some architectures don't mind this and for some very

361 * edge-case performance sensitive applications (like forwarding large volumes

362 * of small packets), setting this DMA offset to 0 will decrease the number of

363 * PCI-E Bus transfers enough to measurably affect performance.

364 */

365 static int rx_dma_offset = 2;

366

367 static bool vf_acls;

368

369 #ifdef CONFIG_PCI_IOV

370 module_param(vf_acls, bool, 0644);

371 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");

372

373 /* Configure the number of PCI-E Virtual Function which are to be instantiated

374 * on SR-IOV Capable Physical Functions.

375 */

376 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];

377

378 module_param_array(num_vf, uint, NULL, 0644);

379 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");

380 #endif

381

382 /*

383 * The filter TCAM has a fixed portion and a variable portion. The fixed

384 * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP

385 * ports. The variable portion is 36 bits which can include things like Exact

386 * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),

387 * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would

388 * far exceed the 36-bit budget for this "compressed" header portion of the

389 * filter. Thus, we have a scarce resource which must be carefully managed.

390 *

391 * By default we set this up to mostly match the set of filter matching

392 * capabilities of T3 but with accommodations for some of T4's more

393 * interesting features:

394 *

395 * { IP Fragment (1), MPS Match Type (3), IP Protocol (8),

396 * [Inner] VLAN (17), Port (3), FCoE (1) }

397 */

398 enum {

399 TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,

400 TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT,

401 TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT,

402 };

403

404 static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;

405

406 module_param(tp_vlan_pri_map, uint, 0644);

407 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration");

408

409 static struct dentry *cxgb4_debugfs_root;

410

411 static LIST_HEAD(adapter_list);

412 static DEFINE_MUTEX(uld_mutex);

413 /* Adapter list to be accessed from atomic context */

414 static LIST_HEAD(adap_rcu_list);

415 static DEFINE_SPINLOCK(adap_rcu_lock);

416 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];

417 static const char *uld_str[] = { "RDMA", "iSCSI" };

418

419 static void link_report(struct net_device *dev)

420 {

421 if (!netif_carrier_ok(dev))

422 netdev_info(dev, "link down\n");

423 else {

424 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };

425

426 const char *s = "10Mbps";

427 const struct port_info *p = netdev_priv(dev);

428

429 switch (p->link_cfg.speed) {

430 case 10000:

431 s = "10Gbps";

432 break;

433 case 1000:

434 s = "1000Mbps";

435 break;

436 case 100:

437 s = "100Mbps";

438 break;

439 case 40000:

440 s = "40Gbps";

441 break;

442 }

443

444 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,

445 fc[p->link_cfg.fc]);

446 }

447 }

448

449 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)

450 {

451 struct net_device *dev = adapter->port[port_id];

452

453 /* Skip changes from disabled ports. */

454 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {

455 if (link_stat)

456 netif_carrier_on(dev);

457 else

458 netif_carrier_off(dev);

459

460 link_report(dev);

461 }

462 }

463

464 void t4_os_portmod_changed(const struct adapter *adap, int port_id)

465 {

466 static const char *mod_str[] = {

467 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"

468 };

469

470 const struct net_device *dev = adap->port[port_id];

471 const struct port_info *pi = netdev_priv(dev);

472

473 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)

474 netdev_info(dev, "port module unplugged\n");

475 else if (pi->mod_type < ARRAY_SIZE(mod_str))

476 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);

477 }

478

479 /*

480 * Configure the exact and hash address filters to handle a port's multicast

481 * and secondary unicast MAC addresses.

482 */

483 static int set_addr_filters(const struct net_device *dev, bool sleep)

484 {

485 u64 mhash = 0;

486 u64 uhash = 0;

487 bool free = true;

488 u16 filt_idx[7];

489 const u8 *addr[7];

490 int ret, naddr = 0;

491 const struct netdev_hw_addr *ha;

492 int uc_cnt = netdev_uc_count(dev);

493 int mc_cnt = netdev_mc_count(dev);

494 const struct port_info *pi = netdev_priv(dev);

495 unsigned int mb = pi->adapter->fn;

496

497 /* first do the secondary unicast addresses */

498 netdev_for_each_uc_addr(ha, dev) {

499 addr[naddr++] = ha->addr;

500 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {

501 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,

502 naddr, addr, filt_idx, &uhash, sleep);

503 if (ret < 0)

504 return ret;

505

506 free = false;

507 naddr = 0;

508 }

509 }

510

511 /* next set up the multicast addresses */

512 netdev_for_each_mc_addr(ha, dev) {

513 addr[naddr++] = ha->addr;

514 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {

515 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,

516 naddr, addr, filt_idx, &mhash, sleep);

517 if (ret < 0)

518 return ret;

519

520 free = false;

521 naddr = 0;

522 }

523 }

524

525 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,

526 uhash | mhash, sleep);

527 }

528

529 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */

530 module_param(dbfifo_int_thresh, int, 0644);

531 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");

532

533 /*

534 * usecs to sleep while draining the dbfifo

535 */

536 static int dbfifo_drain_delay = 1000;

537 module_param(dbfifo_drain_delay, int, 0644);

538 MODULE_PARM_DESC(dbfifo_drain_delay,

539 "usecs to sleep while draining the dbfifo");

540

541 /*

542 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.

543 * If @mtu is -1 it is left unchanged.

544 */

545 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)

546 {

547 int ret;

548 struct port_info *pi = netdev_priv(dev);

549

550 ret = set_addr_filters(dev, sleep_ok);

551 if (ret == 0)

552 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,

553 (dev->flags & IFF_PROMISC) ? 1 : 0,

554 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,

555 sleep_ok);

556 return ret;

557 }

558

559 static struct workqueue_struct *workq;

560

561 /**

562 * link_start - enable a port

563 * @dev: the port to enable

564 *

565 * Performs the MAC and PHY actions needed to enable a port.

566 */

567 static int link_start(struct net_device *dev)

568 {

569 int ret;

570 struct port_info *pi = netdev_priv(dev);

571 unsigned int mb = pi->adapter->fn;

572

573 /*

574 * We do not set address filters and promiscuity here, the stack does

575 * that step explicitly.

576 */

577 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,

578 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);

579 if (ret == 0) {

580 ret = t4_change_mac(pi->adapter, mb, pi->viid,

581 pi->xact_addr_filt, dev->dev_addr, true,

582 true);

583 if (ret >= 0) {

584 pi->xact_addr_filt = ret;

585 ret = 0;

586 }

587 }

588 if (ret == 0)

589 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,

590 &pi->link_cfg);

591 if (ret == 0)

592 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);

593 return ret;

594 }

595

596 /* Clear a filter and release any of its resources that we own. This also

597 * clears the filter's "pending" status.

598 */

599 static void clear_filter(struct adapter *adap, struct filter_entry *f)

600 {

601 /* If the new or old filter have loopback rewriteing rules then we'll

602 * need to free any existing Layer Two Table (L2T) entries of the old

603 * filter rule. The firmware will handle freeing up any Source MAC

604 * Table (SMT) entries used for rewriting Source MAC Addresses in

605 * loopback rules.

606 */

607 if (f->l2t)

608 cxgb4_l2t_release(f->l2t);

609

610 /* The zeroing of the filter rule below clears the filter valid,

611 * pending, locked flags, l2t pointer, etc. so it's all we need for

612 * this operation.

613 */

614 memset(f, 0, sizeof(*f));

615 }

616

617 /* Handle a filter write/deletion reply.

618 */

619 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)

620 {

621 unsigned int idx = GET_TID(rpl);

622 unsigned int nidx = idx - adap->tids.ftid_base;

623 unsigned int ret;

624 struct filter_entry *f;

625

626 if (idx >= adap->tids.ftid_base && nidx <

627 (adap->tids.nftids + adap->tids.nsftids)) {

628 idx = nidx;

629 ret = GET_TCB_COOKIE(rpl->cookie);

630 f = &adap->tids.ftid_tab[idx];

631

632 if (ret == FW_FILTER_WR_FLT_DELETED) {

633 /* Clear the filter when we get confirmation from the

634 * hardware that the filter has been deleted.

635 */

636 clear_filter(adap, f);

637 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {

638 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",

639 idx);

640 clear_filter(adap, f);

641 } else if (ret == FW_FILTER_WR_FLT_ADDED) {

642 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;

643 f->pending = 0; /* asynchronous setup completed */

644 f->valid = 1;

645 } else {

646 /* Something went wrong. Issue a warning about the

647 * problem and clear everything out.

648 */

649 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",

650 idx, ret);

651 clear_filter(adap, f);

652 }

653 }

654 }

655

656 /* Response queue handler for the FW event queue.

657 */

658 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,

659 const struct pkt_gl *gl)

660 {

661 u8 opcode = ((const struct rss_header *)rsp)->opcode;

662

663 rsp++; /* skip RSS header */

664

665 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.

666 */

667 if (unlikely(opcode == CPL_FW4_MSG &&

668 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {

669 rsp++;

670 opcode = ((const struct rss_header *)rsp)->opcode;

671 rsp++;

672 if (opcode != CPL_SGE_EGR_UPDATE) {

673 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"

674 , opcode);

675 goto out;

676 }

677 }

678

679 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {

680 const struct cpl_sge_egr_update *p = (void *)rsp;

681 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));

682 struct sge_txq *txq;

683

684 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];

685 txq->restarts++;

686 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {

687 struct sge_eth_txq *eq;

688

689 eq = container_of(txq, struct sge_eth_txq, q);

690 netif_tx_wake_queue(eq->txq);

691 } else {

692 struct sge_ofld_txq *oq;

693

694 oq = container_of(txq, struct sge_ofld_txq, q);

695 tasklet_schedule(&oq->qresume_tsk);

696 }

697 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {

698 const struct cpl_fw6_msg *p = (void *)rsp;

699

700 if (p->type == 0)

701 t4_handle_fw_rpl(q->adap, p->data);

702 } else if (opcode == CPL_L2T_WRITE_RPL) {

703 const struct cpl_l2t_write_rpl *p = (void *)rsp;

704

705 do_l2t_write_rpl(q->adap, p);

706 } else if (opcode == CPL_SET_TCB_RPL) {

707 const struct cpl_set_tcb_rpl *p = (void *)rsp;

708

709 filter_rpl(q->adap, p);

710 } else

711 dev_err(q->adap->pdev_dev,

712 "unexpected CPL %#x on FW event queue\n", opcode);

713 out:

714 return 0;

715 }

716

717 /**

718 * uldrx_handler - response queue handler for ULD queues

719 * @q: the response queue that received the packet

720 * @rsp: the response queue descriptor holding the offload message

721 * @gl: the gather list of packet fragments

722 *

723 * Deliver an ingress offload packet to a ULD. All processing is done by

724 * the ULD, we just maintain statistics.

725 */

726 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,

727 const struct pkt_gl *gl)

728 {

729 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);

730

731 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.

732 */

733 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&

734 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)

735 rsp += 2;

736

737 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {

738 rxq->stats.nomem++;

739 return -1;

740 }

741 if (gl == NULL)

742 rxq->stats.imm++;

743 else if (gl == CXGB4_MSG_AN)

744 rxq->stats.an++;

745 else

746 rxq->stats.pkts++;

747 return 0;

748 }

749

750 static void disable_msi(struct adapter *adapter)

751 {

752 if (adapter->flags & USING_MSIX) {

753 pci_disable_msix(adapter->pdev);

754 adapter->flags &= ~USING_MSIX;

755 } else if (adapter->flags & USING_MSI) {

756 pci_disable_msi(adapter->pdev);

757 adapter->flags &= ~USING_MSI;

758 }

759 }

760

761 /*

762 * Interrupt handler for non-data events used with MSI-X.

763 */

764 static irqreturn_t t4_nondata_intr(int irq, void *cookie)

765 {

766 struct adapter *adap = cookie;

767

768 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));

769 if (v & PFSW) {

770 adap->swintr = 1;

771 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);

772 }

773 t4_slow_intr_handler(adap);

774 return IRQ_HANDLED;

775 }

776

777 /*

778 * Name the MSI-X interrupts.

779 */

780 static void name_msix_vecs(struct adapter *adap)

781 {

782 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);

783

784 /* non-data interrupts */

785 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);

786

787 /* FW events */

788 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",

789 adap->port[0]->name);

790

791 /* Ethernet queues */

792 for_each_port(adap, j) {

793 struct net_device *d = adap->port[j];

794 const struct port_info *pi = netdev_priv(d);

795

796 for (i = 0; i < pi->nqsets; i++, msi_idx++)

797 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",

798 d->name, i);

799 }

800

801 /* offload queues */

802 for_each_ofldrxq(&adap->sge, i)

803 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",

804 adap->port[0]->name, i);

805

806 for_each_rdmarxq(&adap->sge, i)

807 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",

808 adap->port[0]->name, i);

809 }

810

811 static int request_msix_queue_irqs(struct adapter *adap)

812 {

813 struct sge *s = &adap->sge;

814 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi_index = 2;

815

816 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,

817 adap->msix_info[1].desc, &s->fw_evtq);

818 if (err)

819 return err;

820

821 for_each_ethrxq(s, ethqidx) {

822 err = request_irq(adap->msix_info[msi_index].vec,

823 t4_sge_intr_msix, 0,

824 adap->msix_info[msi_index].desc,

825 &s->ethrxq[ethqidx].rspq);

826 if (err)

827 goto unwind;

828 msi_index++;

829 }

830 for_each_ofldrxq(s, ofldqidx) {

831 err = request_irq(adap->msix_info[msi_index].vec,

832 t4_sge_intr_msix, 0,

833 adap->msix_info[msi_index].desc,

834 &s->ofldrxq[ofldqidx].rspq);

835 if (err)

836 goto unwind;

837 msi_index++;

838 }

839 for_each_rdmarxq(s, rdmaqidx) {

840 err = request_irq(adap->msix_info[msi_index].vec,

841 t4_sge_intr_msix, 0,

842 adap->msix_info[msi_index].desc,

843 &s->rdmarxq[rdmaqidx].rspq);

844 if (err)

845 goto unwind;

846 msi_index++;

847 }

848 return 0;

849

850 unwind:

851 while (--rdmaqidx >= 0)

852 free_irq(adap->msix_info[--msi_index].vec,

853 &s->rdmarxq[rdmaqidx].rspq);

854 while (--ofldqidx >= 0)

855 free_irq(adap->msix_info[--msi_index].vec,

856 &s->ofldrxq[ofldqidx].rspq);

857 while (--ethqidx >= 0)

858 free_irq(adap->msix_info[--msi_index].vec,

859 &s->ethrxq[ethqidx].rspq);

860 free_irq(adap->msix_info[1].vec, &s->fw_evtq);

861 return err;

862 }

863

864 static void free_msix_queue_irqs(struct adapter *adap)

865 {

866 int i, msi_index = 2;

867 struct sge *s = &adap->sge;

868

869 free_irq(adap->msix_info[1].vec, &s->fw_evtq);

870 for_each_ethrxq(s, i)

871 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);

872 for_each_ofldrxq(s, i)

873 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);

874 for_each_rdmarxq(s, i)

875 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);

876 }

877

878 /**

879 * write_rss - write the RSS table for a given port

880 * @pi: the port

881 * @queues: array of queue indices for RSS

882 *

883 * Sets up the portion of the HW RSS table for the port's VI to distribute

884 * packets to the Rx queues in @queues.

885 */

886 static int write_rss(const struct port_info *pi, const u16 *queues)

887 {

888 u16 *rss;

889 int i, err;

890 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];

891

892 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);

893 if (!rss)

894 return -ENOMEM;

895

896 /* map the queue indices to queue ids */

897 for (i = 0; i < pi->rss_size; i++, queues++)

898 rss[i] = q[*queues].rspq.abs_id;

899

900 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,

901 pi->rss_size, rss, pi->rss_size);

902 kfree(rss);

903 return err;

904 }

905

906 /**

907 * setup_rss - configure RSS

908 * @adap: the adapter

909 *

910 * Sets up RSS for each port.

911 */

912 static int setup_rss(struct adapter *adap)

913 {

914 int i, err;

915

916 for_each_port(adap, i) {

917 const struct port_info *pi = adap2pinfo(adap, i);

918

919 err = write_rss(pi, pi->rss);

920 if (err)

921 return err;

922 }

923 return 0;

924 }

925

926 /*

927 * Return the channel of the ingress queue with the given qid.

928 */

929 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)

930 {

931 qid -= p->ingr_start;

932 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;

933 }

934

935 /*

936 * Wait until all NAPI handlers are descheduled.

937 */

938 static void quiesce_rx(struct adapter *adap)

939 {

940 int i;

941

942 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {

943 struct sge_rspq *q = adap->sge.ingr_map[i];

944

945 if (q && q->handler)

946 napi_disable(&q->napi);

947 }

948 }

949

950 /*

951 * Enable NAPI scheduling and interrupt generation for all Rx queues.

952 */

953 static void enable_rx(struct adapter *adap)

954 {

955 int i;

956

957 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {

958 struct sge_rspq *q = adap->sge.ingr_map[i];

959

960 if (!q)

961 continue;

962 if (q->handler)

963 napi_enable(&q->napi);

964 /* 0-increment GTS to start the timer and enable interrupts */

965 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),

966 SEINTARM(q->intr_params) |

967 INGRESSQID(q->cntxt_id));

968 }

969 }

970

971 /**

972 * setup_sge_queues - configure SGE Tx/Rx/response queues

973 * @adap: the adapter

974 *

975 * Determines how many sets of SGE queues to use and initializes them.

976 * We support multiple queue sets per port if we have MSI-X, otherwise

977 * just one queue set per port.

978 */

979 static int setup_sge_queues(struct adapter *adap)

980 {

981 int err, msi_idx, i, j;

982 struct sge *s = &adap->sge;

983

984 bitmap_zero(s->starving_fl, MAX_EGRQ);

985 bitmap_zero(s->txq_maperr, MAX_EGRQ);

986

987 if (adap->flags & USING_MSIX)

988 msi_idx = 1; /* vector 0 is for non-queue interrupts */

989 else {

990 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,

991 NULL, NULL);

992 if (err)

993 return err;

994 msi_idx = -((int)s->intrq.abs_id + 1);

995 }

996

997 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],

998 msi_idx, NULL, fwevtq_handler);

999 if (err) {

1000 freeout: t4_free_sge_resources(adap);

1001 return err;

1002 }

1003

1004 for_each_port(adap, i) {

1005 struct net_device *dev = adap->port[i];

1006 struct port_info *pi = netdev_priv(dev);

1007 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];

1008 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];

1009

1010 for (j = 0; j < pi->nqsets; j++, q++) {

1011 if (msi_idx > 0)

1012 msi_idx++;

1013 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,

1014 msi_idx, &q->fl,

1015 t4_ethrx_handler);

1016 if (err)

1017 goto freeout;

1018 q->rspq.idx = j;

1019 memset(&q->stats, 0, sizeof(q->stats));

1020 }

1021 for (j = 0; j < pi->nqsets; j++, t++) {

1022 err = t4_sge_alloc_eth_txq(adap, t, dev,

1023 netdev_get_tx_queue(dev, j),

1024 s->fw_evtq.cntxt_id);

1025 if (err)

1026 goto freeout;

1027 }

1028 }

1029

1030 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */

1031 for_each_ofldrxq(s, i) {

1032 struct sge_ofld_rxq *q = &s->ofldrxq[i];

1033 struct net_device *dev = adap->port[i / j];

1034

1035 if (msi_idx > 0)

1036 msi_idx++;

1037 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,

1038 &q->fl, uldrx_handler);

1039 if (err)

1040 goto freeout;

1041 memset(&q->stats, 0, sizeof(q->stats));

1042 s->ofld_rxq[i] = q->rspq.abs_id;

1043 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,

1044 s->fw_evtq.cntxt_id);

1045 if (err)

1046 goto freeout;

1047 }

1048

1049 for_each_rdmarxq(s, i) {

1050 struct sge_ofld_rxq *q = &s->rdmarxq[i];

1051

1052 if (msi_idx > 0)

1053 msi_idx++;

1054 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],

1055 msi_idx, &q->fl, uldrx_handler);

1056 if (err)

1057 goto freeout;

1058 memset(&q->stats, 0, sizeof(q->stats));

1059 s->rdma_rxq[i] = q->rspq.abs_id;

1060 }

1061

1062 for_each_port(adap, i) {

1063 /*

1064 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't

1065 * have RDMA queues, and that's the right value.

1066 */

1067 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],

1068 s->fw_evtq.cntxt_id,

1069 s->rdmarxq[i].rspq.cntxt_id);

1070 if (err)

1071 goto freeout;

1072 }

1073

1074 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,

1075 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |

1076 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));

1077 return 0;

1078 }

1079

1080 /*

1081 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.

1082 * The allocated memory is cleared.

1083 */

1084 void *t4_alloc_mem(size_t size)

1085 {

1086 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);

1087

1088 if (!p)

1089 p = vzalloc(size);

1090 return p;

1091 }

1092

1093 /*

1094 * Free memory allocated through alloc_mem().

1095 */

1096 static void t4_free_mem(void *addr)

1097 {

1098 if (is_vmalloc_addr(addr))

1099 vfree(addr);

1100 else

1101 kfree(addr);

1102 }

1103

1104 /* Send a Work Request to write the filter at a specified index. We construct

1105 * a Firmware Filter Work Request to have the work done and put the indicated

1106 * filter into "pending" mode which will prevent any further actions against

1107 * it till we get a reply from the firmware on the completion status of the

1108 * request.

1109 */

1110 static int set_filter_wr(struct adapter *adapter, int fidx)

1111 {

1112 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];

1113 struct sk_buff *skb;

1114 struct fw_filter_wr *fwr;

1115 unsigned int ftid;

1116

1117 /* If the new filter requires loopback Destination MAC and/or VLAN

1118 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for

1119 * the filter.

1120 */

1121 if (f->fs.newdmac || f->fs.newvlan) {

1122 /* allocate L2T entry for new filter */

1123 f->l2t = t4_l2t_alloc_switching(adapter->l2t);

1124 if (f->l2t == NULL)

1125 return -EAGAIN;

1126 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,

1127 f->fs.eport, f->fs.dmac)) {

1128 cxgb4_l2t_release(f->l2t);

1129 f->l2t = NULL;

1130 return -ENOMEM;

1131 }

1132 }

1133

1134 ftid = adapter->tids.ftid_base + fidx;

1135

1136 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);

1137 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));

1138 memset(fwr, 0, sizeof(*fwr));

1139

1140 /* It would be nice to put most of the following in t4_hw.c but most

1141 * of the work is translating the cxgbtool ch_filter_specification

1142 * into the Work Request and the definition of that structure is

1143 * currently in cxgbtool.h which isn't appropriate to pull into the

1144 * common code. We may eventually try to come up with a more neutral

1145 * filter specification structure but for now it's easiest to simply

1146 * put this fairly direct code in line ...

1147 */

1148 fwr->op_pkd = htonl(FW_WR_OP(FW_FILTER_WR));

1149 fwr->len16_pkd = htonl(FW_WR_LEN16(sizeof(*fwr)/16));

1150 fwr->tid_to_iq =

1151 htonl(V_FW_FILTER_WR_TID(ftid) |

1152 V_FW_FILTER_WR_RQTYPE(f->fs.type) |

1153 V_FW_FILTER_WR_NOREPLY(0) |

1154 V_FW_FILTER_WR_IQ(f->fs.iq));

1155 fwr->del_filter_to_l2tix =

1156 htonl(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |

1157 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |

1158 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |

1159 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |

1160 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |

1161 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |

1162 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |

1163 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |

1164 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||

1165 f->fs.newvlan == VLAN_REWRITE) |

1166 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||

1167 f->fs.newvlan == VLAN_REWRITE) |

1168 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |

1169 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |

1170 V_FW_FILTER_WR_PRIO(f->fs.prio) |

1171 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));

1172 fwr->ethtype = htons(f->fs.val.ethtype);

1173 fwr->ethtypem = htons(f->fs.mask.ethtype);

1174 fwr->frag_to_ovlan_vldm =

1175 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |

1176 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |

1177 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.ivlan_vld) |

1178 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.ovlan_vld) |

1179 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.ivlan_vld) |

1180 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.ovlan_vld));

1181 fwr->smac_sel = 0;

1182 fwr->rx_chan_rx_rpl_iq =

1183 htons(V_FW_FILTER_WR_RX_CHAN(0) |

1184 V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id));

1185 fwr->maci_to_matchtypem =

1186 htonl(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |

1187 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |

1188 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |

1189 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |

1190 V_FW_FILTER_WR_PORT(f->fs.val.iport) |

1191 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |

1192 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |

1193 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));

1194 fwr->ptcl = f->fs.val.proto;

1195 fwr->ptclm = f->fs.mask.proto;

1196 fwr->ttyp = f->fs.val.tos;

1197 fwr->ttypm = f->fs.mask.tos;

1198 fwr->ivlan = htons(f->fs.val.ivlan);

1199 fwr->ivlanm = htons(f->fs.mask.ivlan);

1200 fwr->ovlan = htons(f->fs.val.ovlan);

1201 fwr->ovlanm = htons(f->fs.mask.ovlan);

1202 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));

1203 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));

1204 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));

1205 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));

1206 fwr->lp = htons(f->fs.val.lport);

1207 fwr->lpm = htons(f->fs.mask.lport);

1208 fwr->fp = htons(f->fs.val.fport);

1209 fwr->fpm = htons(f->fs.mask.fport);

1210 if (f->fs.newsmac)

1211 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));

1212

1213 /* Mark the filter as "pending" and ship off the Filter Work Request.

1214 * When we get the Work Request Reply we'll clear the pending status.

1215 */

1216 f->pending = 1;

1217 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);

1218 t4_ofld_send(adapter, skb);

1219 return 0;

1220 }

1221

1222 /* Delete the filter at a specified index.

1223 */

1224 static int del_filter_wr(struct adapter *adapter, int fidx)

1225 {

1226 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];

1227 struct sk_buff *skb;

1228 struct fw_filter_wr *fwr;

1229 unsigned int len, ftid;

1230

1231 len = sizeof(*fwr);

1232 ftid = adapter->tids.ftid_base + fidx;

1233

1234 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);

1235 fwr = (struct fw_filter_wr *)__skb_put(skb, len);

1236 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);

1237

1238 /* Mark the filter as "pending" and ship off the Filter Work Request.

1239 * When we get the Work Request Reply we'll clear the pending status.

1240 */

1241 f->pending = 1;

1242 t4_mgmt_tx(adapter, skb);

1243 return 0;

1244 }

1245

1246 static inline int is_offload(const struct adapter *adap)

1247 {

1248 return adap->params.offload;

1249 }

1250

1251 /*

1252 * Implementation of ethtool operations.

1253 */

1254

1255 static u32 get_msglevel(struct net_device *dev)

1256 {

1257 return netdev2adap(dev)->msg_enable;

1258 }

1259

1260 static void set_msglevel(struct net_device *dev, u32 val)

1261 {

1262 netdev2adap(dev)->msg_enable = val;

1263 }

1264

1265 static char stats_strings[][ETH_GSTRING_LEN] = {

1266 "TxOctetsOK ",

1267 "TxFramesOK ",

1268 "TxBroadcastFrames ",

1269 "TxMulticastFrames ",

1270 "TxUnicastFrames ",

1271 "TxErrorFrames ",

1272

1273 "TxFrames64 ",

1274 "TxFrames65To127 ",

1275 "TxFrames128To255 ",

1276 "TxFrames256To511 ",

1277 "TxFrames512To1023 ",

1278 "TxFrames1024To1518 ",

1279 "TxFrames1519ToMax ",

1280

1281 "TxFramesDropped ",

1282 "TxPauseFrames ",

1283 "TxPPP0Frames ",

1284 "TxPPP1Frames ",

1285 "TxPPP2Frames ",

1286 "TxPPP3Frames ",

1287 "TxPPP4Frames ",

1288 "TxPPP5Frames ",

1289 "TxPPP6Frames ",

1290 "TxPPP7Frames ",

1291

1292 "RxOctetsOK ",

1293 "RxFramesOK ",

1294 "RxBroadcastFrames ",

1295 "RxMulticastFrames ",

1296 "RxUnicastFrames ",

1297

1298 "RxFramesTooLong ",

1299 "RxJabberErrors ",

1300 "RxFCSErrors ",

1301 "RxLengthErrors ",

1302 "RxSymbolErrors ",

1303 "RxRuntFrames ",

1304

1305 "RxFrames64 ",

1306 "RxFrames65To127 ",

1307 "RxFrames128To255 ",

1308 "RxFrames256To511 ",

1309 "RxFrames512To1023 ",

1310 "RxFrames1024To1518 ",

1311 "RxFrames1519ToMax ",

1312

1313 "RxPauseFrames ",

1314 "RxPPP0Frames ",

1315 "RxPPP1Frames ",

1316 "RxPPP2Frames ",

1317 "RxPPP3Frames ",

1318 "RxPPP4Frames ",

1319 "RxPPP5Frames ",

1320 "RxPPP6Frames ",

1321 "RxPPP7Frames ",

1322

1323 "RxBG0FramesDropped ",

1324 "RxBG1FramesDropped ",

1325 "RxBG2FramesDropped ",

1326 "RxBG3FramesDropped ",

1327 "RxBG0FramesTrunc ",

1328 "RxBG1FramesTrunc ",

1329 "RxBG2FramesTrunc ",

1330 "RxBG3FramesTrunc ",

1331

1332 "TSO ",

1333 "TxCsumOffload ",

1334 "RxCsumGood ",

1335 "VLANextractions ",

1336 "VLANinsertions ",

1337 "GROpackets ",

1338 "GROmerged ",

1339 "WriteCoalSuccess ",

1340 "WriteCoalFail ",

1341 };

1342

1343 static int get_sset_count(struct net_device *dev, int sset)

1344 {

1345 switch (sset) {

1346 case ETH_SS_STATS:

1347 return ARRAY_SIZE(stats_strings);

1348 default:

1349 return -EOPNOTSUPP;

1350 }

1351 }

1352

1353 #define T4_REGMAP_SIZE (160 * 1024)

1354 #define T5_REGMAP_SIZE (332 * 1024)

1355

1356 static int get_regs_len(struct net_device *dev)

1357 {

1358 struct adapter *adap = netdev2adap(dev);

1359 if (is_t4(adap->params.chip))

1360 return T4_REGMAP_SIZE;

1361 else

1362 return T5_REGMAP_SIZE;

1363 }

1364

1365 static int get_eeprom_len(struct net_device *dev)

1366 {

1367 return EEPROMSIZE;

1368 }

1369

1370 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)

1371 {

1372 struct adapter *adapter = netdev2adap(dev);

1373

1374 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));

1375 strlcpy(info->version, DRV_VERSION, sizeof(info->version));

1376 strlcpy(info->bus_info, pci_name(adapter->pdev),

1377 sizeof(info->bus_info));

1378

1379 if (adapter->params.fw_vers)

1380 snprintf(info->fw_version, sizeof(info->fw_version),

1381 "%u.%u.%u.%u, TP %u.%u.%u.%u",

1382 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),

1383 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),

1384 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),

1385 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),

1386 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),

1387 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),

1388 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),

1389 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));

1390 }

1391

1392 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)

1393 {

1394 if (stringset == ETH_SS_STATS)

1395 memcpy(data, stats_strings, sizeof(stats_strings));

1396 }

1397

1398 /*

1399 * port stats maintained per queue of the port. They should be in the same

1400 * order as in stats_strings above.

1401 */

1402 struct queue_port_stats {

1403 u64 tso;

1404 u64 tx_csum;

1405 u64 rx_csum;

1406 u64 vlan_ex;

1407 u64 vlan_ins;

1408 u64 gro_pkts;

1409 u64 gro_merged;

1410 };

1411

1412 static void collect_sge_port_stats(const struct adapter *adap,

1413 const struct port_info *p, struct queue_port_stats *s)

1414 {

1415 int i;

1416 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];

1417 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];

1418

1419 memset(s, 0, sizeof(*s));

1420 for (i = 0; i < p->nqsets; i++, rx++, tx++) {

1421 s->tso += tx->tso;

1422 s->tx_csum += tx->tx_cso;

1423 s->rx_csum += rx->stats.rx_cso;

1424 s->vlan_ex += rx->stats.vlan_ex;

1425 s->vlan_ins += tx->vlan_ins;

1426 s->gro_pkts += rx->stats.lro_pkts;

1427 s->gro_merged += rx->stats.lro_merged;

1428 }

1429 }

1430

1431 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,

1432 u64 *data)

1433 {

1434 struct port_info *pi = netdev_priv(dev);

1435 struct adapter *adapter = pi->adapter;

1436 u32 val1, val2;

1437

1438 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);

1439

1440 data += sizeof(struct port_stats) / sizeof(u64);

1441 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);

1442 data += sizeof(struct queue_port_stats) / sizeof(u64);

1443 if (!is_t4(adapter->params.chip)) {

1444 t4_write_reg(adapter, SGE_STAT_CFG, STATSOURCE_T5(7));

1445 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL);

1446 val2 = t4_read_reg(adapter, SGE_STAT_MATCH);

1447 *data = val1 - val2;

1448 data++;

1449 *data = val2;

1450 data++;

1451 } else {

1452 memset(data, 0, 2 * sizeof(u64));

1453 *data += 2;

1454 }

1455 }

1456

1457 /*

1458 * Return a version number to identify the type of adapter. The scheme is:

1459 * - bits 0..9: chip version

1460 * - bits 10..15: chip revision

1461 * - bits 16..23: register dump version

1462 */

1463 static inline unsigned int mk_adap_vers(const struct adapter *ap)

1464 {

1465 return CHELSIO_CHIP_VERSION(ap->params.chip) |

1466 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);

1467 }

1468

1469 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,

1470 unsigned int end)

1471 {

1472 u32 *p = buf + start;

1473

1474 for ( ; start <= end; start += sizeof(u32))

1475 *p++ = t4_read_reg(ap, start);

1476 }

1477

1478 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,

1479 void *buf)

1480 {

1481 static const unsigned int t4_reg_ranges[] = {

1482 0x1008, 0x1108,

1483 0x1180, 0x11b4,

1484 0x11fc, 0x123c,

1485 0x1300, 0x173c,

1486 0x1800, 0x18fc,

1487 0x3000, 0x30d8,

1488 0x30e0, 0x5924,

1489 0x5960, 0x59d4,

1490 0x5a00, 0x5af8,

1491 0x6000, 0x6098,

1492 0x6100, 0x6150,

1493 0x6200, 0x6208,

1494 0x6240, 0x6248,

1495 0x6280, 0x6338,

1496 0x6370, 0x638c,

1497 0x6400, 0x643c,

1498 0x6500, 0x6524,

1499 0x6a00, 0x6a38,

1500 0x6a60, 0x6a78,

1501 0x6b00, 0x6b84,

1502 0x6bf0, 0x6c84,

1503 0x6cf0, 0x6d84,

1504 0x6df0, 0x6e84,

1505 0x6ef0, 0x6f84,

1506 0x6ff0, 0x7084,

1507 0x70f0, 0x7184,

1508 0x71f0, 0x7284,

1509 0x72f0, 0x7384,

1510 0x73f0, 0x7450,

1511 0x7500, 0x7530,

1512 0x7600, 0x761c,

1513 0x7680, 0x76cc,

1514 0x7700, 0x7798,

1515 0x77c0, 0x77fc,

1516 0x7900, 0x79fc,

1517 0x7b00, 0x7c38,

1518 0x7d00, 0x7efc,

1519 0x8dc0, 0x8e1c,

1520 0x8e30, 0x8e78,

1521 0x8ea0, 0x8f6c,

1522 0x8fc0, 0x9074,

1523 0x90fc, 0x90fc,

1524 0x9400, 0x9458,

1525 0x9600, 0x96bc,

1526 0x9800, 0x9808,

1527 0x9820, 0x983c,

1528 0x9850, 0x9864,

1529 0x9c00, 0x9c6c,

1530 0x9c80, 0x9cec,

1531 0x9d00, 0x9d6c,

1532 0x9d80, 0x9dec,

1533 0x9e00, 0x9e6c,

1534 0x9e80, 0x9eec,

1535 0x9f00, 0x9f6c,

1536 0x9f80, 0x9fec,

1537 0xd004, 0xd03c,

1538 0xdfc0, 0xdfe0,

1539 0xe000, 0xea7c,

1540 0xf000, 0x11190,

1541 0x19040, 0x1906c,

1542 0x19078, 0x19080,

1543 0x1908c, 0x19124,

1544 0x19150, 0x191b0,

1545 0x191d0, 0x191e8,

1546 0x19238, 0x1924c,

1547 0x193f8, 0x19474,

1548 0x19490, 0x194f8,

1549 0x19800, 0x19f30,

1550 0x1a000, 0x1a06c,

1551 0x1a0b0, 0x1a120,

1552 0x1a128, 0x1a138,

1553 0x1a190, 0x1a1c4,

1554 0x1a1fc, 0x1a1fc,

1555 0x1e040, 0x1e04c,

1556 0x1e284, 0x1e28c,

1557 0x1e2c0, 0x1e2c0,

1558 0x1e2e0, 0x1e2e0,

1559 0x1e300, 0x1e384,

1560 0x1e3c0, 0x1e3c8,

1561 0x1e440, 0x1e44c,

1562 0x1e684, 0x1e68c,

1563 0x1e6c0, 0x1e6c0,

1564 0x1e6e0, 0x1e6e0,

1565 0x1e700, 0x1e784,

1566 0x1e7c0, 0x1e7c8,

1567 0x1e840, 0x1e84c,

1568 0x1ea84, 0x1ea8c,

1569 0x1eac0, 0x1eac0,

1570 0x1eae0, 0x1eae0,

1571 0x1eb00, 0x1eb84,

1572 0x1ebc0, 0x1ebc8,

1573 0x1ec40, 0x1ec4c,

1574 0x1ee84, 0x1ee8c,

1575 0x1eec0, 0x1eec0,

1576 0x1eee0, 0x1eee0,

1577 0x1ef00, 0x1ef84,

1578 0x1efc0, 0x1efc8,

1579 0x1f040, 0x1f04c,

1580 0x1f284, 0x1f28c,

1581 0x1f2c0, 0x1f2c0,

1582 0x1f2e0, 0x1f2e0,

1583 0x1f300, 0x1f384,

1584 0x1f3c0, 0x1f3c8,

1585 0x1f440, 0x1f44c,

1586 0x1f684, 0x1f68c,

1587 0x1f6c0, 0x1f6c0,

1588 0x1f6e0, 0x1f6e0,

1589 0x1f700, 0x1f784,

1590 0x1f7c0, 0x1f7c8,

1591 0x1f840, 0x1f84c,

1592 0x1fa84, 0x1fa8c,

1593 0x1fac0, 0x1fac0,

1594 0x1fae0, 0x1fae0,

1595 0x1fb00, 0x1fb84,

1596 0x1fbc0, 0x1fbc8,

1597 0x1fc40, 0x1fc4c,

1598 0x1fe84, 0x1fe8c,

1599 0x1fec0, 0x1fec0,

1600 0x1fee0, 0x1fee0,

1601 0x1ff00, 0x1ff84,

1602 0x1ffc0, 0x1ffc8,

1603 0x20000, 0x2002c,

1604 0x20100, 0x2013c,

1605 0x20190, 0x201c8,

1606 0x20200, 0x20318,

1607 0x20400, 0x20528,

1608 0x20540, 0x20614,

1609 0x21000, 0x21040,

1610 0x2104c, 0x21060,

1611 0x210c0, 0x210ec,

1612 0x21200, 0x21268,

1613 0x21270, 0x21284,

1614 0x212fc, 0x21388,

1615 0x21400, 0x21404,

1616 0x21500, 0x21518,

1617 0x2152c, 0x2153c,

1618 0x21550, 0x21554,

1619 0x21600, 0x21600,

1620 0x21608, 0x21628,

1621 0x21630, 0x2163c,

1622 0x21700, 0x2171c,

1623 0x21780, 0x2178c,

1624 0x21800, 0x21c38,

1625 0x21c80, 0x21d7c,

1626 0x21e00, 0x21e04,

1627 0x22000, 0x2202c,

1628 0x22100, 0x2213c,

1629 0x22190, 0x221c8,

1630 0x22200, 0x22318,

1631 0x22400, 0x22528,

1632 0x22540, 0x22614,

1633 0x23000, 0x23040,

1634 0x2304c, 0x23060,

1635 0x230c0, 0x230ec,

1636 0x23200, 0x23268,

1637 0x23270, 0x23284,

1638 0x232fc, 0x23388,

1639 0x23400, 0x23404,

1640 0x23500, 0x23518,

1641 0x2352c, 0x2353c,

1642 0x23550, 0x23554,

1643 0x23600, 0x23600,

1644 0x23608, 0x23628,

1645 0x23630, 0x2363c,

1646 0x23700, 0x2371c,

1647 0x23780, 0x2378c,

1648 0x23800, 0x23c38,

1649 0x23c80, 0x23d7c,

1650 0x23e00, 0x23e04,

1651 0x24000, 0x2402c,

1652 0x24100, 0x2413c,

1653 0x24190, 0x241c8,

1654 0x24200, 0x24318,

1655 0x24400, 0x24528,

1656 0x24540, 0x24614,

1657 0x25000, 0x25040,

1658 0x2504c, 0x25060,

1659 0x250c0, 0x250ec,

1660 0x25200, 0x25268,

1661 0x25270, 0x25284,

1662 0x252fc, 0x25388,

1663 0x25400, 0x25404,

1664 0x25500, 0x25518,

1665 0x2552c, 0x2553c,

1666 0x25550, 0x25554,

1667 0x25600, 0x25600,

1668 0x25608, 0x25628,

1669 0x25630, 0x2563c,

1670 0x25700, 0x2571c,

1671 0x25780, 0x2578c,

1672 0x25800, 0x25c38,

1673 0x25c80, 0x25d7c,

1674 0x25e00, 0x25e04,

1675 0x26000, 0x2602c,

1676 0x26100, 0x2613c,

1677 0x26190, 0x261c8,

1678 0x26200, 0x26318,

1679 0x26400, 0x26528,

1680 0x26540, 0x26614,

1681 0x27000, 0x27040,

1682 0x2704c, 0x27060,

1683 0x270c0, 0x270ec,

1684 0x27200, 0x27268,

1685 0x27270, 0x27284,

1686 0x272fc, 0x27388,

1687 0x27400, 0x27404,

1688 0x27500, 0x27518,

1689 0x2752c, 0x2753c,

1690 0x27550, 0x27554,

1691 0x27600, 0x27600,

1692 0x27608, 0x27628,

1693 0x27630, 0x2763c,

1694 0x27700, 0x2771c,

1695 0x27780, 0x2778c,

1696 0x27800, 0x27c38,

1697 0x27c80, 0x27d7c,

1698 0x27e00, 0x27e04

1699 };

1700

1701 static const unsigned int t5_reg_ranges[] = {

1702 0x1008, 0x1148,

1703 0x1180, 0x11b4,

1704 0x11fc, 0x123c,

1705 0x1280, 0x173c,

1706 0x1800, 0x18fc,

1707 0x3000, 0x3028,

1708 0x3060, 0x30d8,

1709 0x30e0, 0x30fc,

1710 0x3140, 0x357c,

1711 0x35a8, 0x35cc,

1712 0x35ec, 0x35ec,

1713 0x3600, 0x5624,

1714 0x56cc, 0x575c,

1715 0x580c, 0x5814,

1716 0x5890, 0x58bc,

1717 0x5940, 0x59dc,

1718 0x59fc, 0x5a18,

1719 0x5a60, 0x5a9c,

1720 0x5b9c, 0x5bfc,

1721 0x6000, 0x6040,

1722 0x6058, 0x614c,

1723 0x7700, 0x7798,

1724 0x77c0, 0x78fc,

1725 0x7b00, 0x7c54,

1726 0x7d00, 0x7efc,

1727 0x8dc0, 0x8de0,

1728 0x8df8, 0x8e84,

1729 0x8ea0, 0x8f84,

1730 0x8fc0, 0x90f8,

1731 0x9400, 0x9470,

1732 0x9600, 0x96f4,

1733 0x9800, 0x9808,

1734 0x9820, 0x983c,

1735 0x9850, 0x9864,

1736 0x9c00, 0x9c6c,

1737 0x9c80, 0x9cec,

1738 0x9d00, 0x9d6c,

1739 0x9d80, 0x9dec,

1740 0x9e00, 0x9e6c,

1741 0x9e80, 0x9eec,

1742 0x9f00, 0x9f6c,

1743 0x9f80, 0xa020,

1744 0xd004, 0xd03c,

1745 0xdfc0, 0xdfe0,

1746 0xe000, 0x11088,

1747 0x1109c, 0x1117c,

1748 0x11190, 0x11204,

1749 0x19040, 0x1906c,

1750 0x19078, 0x19080,

1751 0x1908c, 0x19124,

1752 0x19150, 0x191b0,

1753 0x191d0, 0x191e8,

1754 0x19238, 0x19290,

1755 0x193f8, 0x19474,

1756 0x19490, 0x194cc,

1757 0x194f0, 0x194f8,

1758 0x19c00, 0x19c60,

1759 0x19c94, 0x19e10,

1760 0x19e50, 0x19f34,

1761 0x19f40, 0x19f50,

1762 0x19f90, 0x19fe4,

1763 0x1a000, 0x1a06c,

1764 0x1a0b0, 0x1a120,

1765 0x1a128, 0x1a138,

1766 0x1a190, 0x1a1c4,

1767 0x1a1fc, 0x1a1fc,

1768 0x1e008, 0x1e00c,

1769 0x1e040, 0x1e04c,

1770 0x1e284, 0x1e290,

1771 0x1e2c0, 0x1e2c0,

1772 0x1e2e0, 0x1e2e0,

1773 0x1e300, 0x1e384,

1774 0x1e3c0, 0x1e3c8,

1775 0x1e408, 0x1e40c,

1776 0x1e440, 0x1e44c,

1777 0x1e684, 0x1e690,

1778 0x1e6c0, 0x1e6c0,

1779 0x1e6e0, 0x1e6e0,

1780 0x1e700, 0x1e784,

1781 0x1e7c0, 0x1e7c8,

1782 0x1e808, 0x1e80c,

1783 0x1e840, 0x1e84c,

1784 0x1ea84, 0x1ea90,

1785 0x1eac0, 0x1eac0,

1786 0x1eae0, 0x1eae0,

1787 0x1eb00, 0x1eb84,

1788 0x1ebc0, 0x1ebc8,

1789 0x1ec08, 0x1ec0c,

1790 0x1ec40, 0x1ec4c,

1791 0x1ee84, 0x1ee90,

1792 0x1eec0, 0x1eec0,

1793 0x1eee0, 0x1eee0,

1794 0x1ef00, 0x1ef84,

1795 0x1efc0, 0x1efc8,

1796 0x1f008, 0x1f00c,

1797 0x1f040, 0x1f04c,

1798 0x1f284, 0x1f290,

1799 0x1f2c0, 0x1f2c0,

1800 0x1f2e0, 0x1f2e0,

1801 0x1f300, 0x1f384,

1802 0x1f3c0, 0x1f3c8,

1803 0x1f408, 0x1f40c,

1804 0x1f440, 0x1f44c,

1805 0x1f684, 0x1f690,

1806 0x1f6c0, 0x1f6c0,

1807 0x1f6e0, 0x1f6e0,

1808 0x1f700, 0x1f784,

1809 0x1f7c0, 0x1f7c8,

1810 0x1f808, 0x1f80c,

1811 0x1f840, 0x1f84c,

1812 0x1fa84, 0x1fa90,

1813 0x1fac0, 0x1fac0,

1814 0x1fae0, 0x1fae0,

1815 0x1fb00, 0x1fb84,

1816 0x1fbc0, 0x1fbc8,

1817 0x1fc08, 0x1fc0c,

1818 0x1fc40, 0x1fc4c,

1819 0x1fe84, 0x1fe90,

1820 0x1fec0, 0x1fec0,

1821 0x1fee0, 0x1fee0,

1822 0x1ff00, 0x1ff84,

1823 0x1ffc0, 0x1ffc8,

1824 0x30000, 0x30030,

1825 0x30100, 0x30144,

1826 0x30190, 0x301d0,

1827 0x30200, 0x30318,

1828 0x30400, 0x3052c,

1829 0x30540, 0x3061c,

1830 0x30800, 0x30834,

1831 0x308c0, 0x30908,

1832 0x30910, 0x309ac,

1833 0x30a00, 0x30a04,

1834 0x30a0c, 0x30a2c,

1835 0x30a44, 0x30a50,

1836 0x30a74, 0x30c24,

1837 0x30d08, 0x30d14,

1838 0x30d1c, 0x30d20,

1839 0x30d3c, 0x30d50,

1840 0x31200, 0x3120c,

1841 0x31220, 0x31220,

1842 0x31240, 0x31240,

1843 0x31600, 0x31600,

1844 0x31608, 0x3160c,

1845 0x31a00, 0x31a1c,

1846 0x31e04, 0x31e20,

1847 0x31e38, 0x31e3c,

1848 0x31e80, 0x31e80,

1849 0x31e88, 0x31ea8,

1850 0x31eb0, 0x31eb4,

1851 0x31ec8, 0x31ed4,

1852 0x31fb8, 0x32004,

1853 0x32208, 0x3223c,

1854 0x32600, 0x32630,

1855 0x32a00, 0x32abc,

1856 0x32b00, 0x32b70,

1857 0x33000, 0x33048,

1858 0x33060, 0x3309c,

1859 0x330f0, 0x33148,

1860 0x33160, 0x3319c,

1861 0x331f0, 0x332e4,

1862 0x332f8, 0x333e4,

1863 0x333f8, 0x33448,

1864 0x33460, 0x3349c,

1865 0x334f0, 0x33548,

1866 0x33560, 0x3359c,

1867 0x335f0, 0x336e4,

1868 0x336f8, 0x337e4,

1869 0x337f8, 0x337fc,

1870 0x33814, 0x33814,

1871 0x3382c, 0x3382c,

1872 0x33880, 0x3388c,

1873 0x338e8, 0x338ec,

1874 0x33900, 0x33948,

1875 0x33960, 0x3399c,

1876 0x339f0, 0x33ae4,

1877 0x33af8, 0x33b10,

1878 0x33b28, 0x33b28,

1879 0x33b3c, 0x33b50,

1880 0x33bf0, 0x33c10,

1881 0x33c28, 0x33c28,

1882 0x33c3c, 0x33c50,

1883 0x33cf0, 0x33cfc,

1884 0x34000, 0x34030,

1885 0x34100, 0x34144,

1886 0x34190, 0x341d0,

1887 0x34200, 0x34318,

1888 0x34400, 0x3452c,

1889 0x34540, 0x3461c,

1890 0x34800, 0x34834,

1891 0x348c0, 0x34908,

1892 0x34910, 0x349ac,

1893 0x34a00, 0x34a04,

1894 0x34a0c, 0x34a2c,

1895 0x34a44, 0x34a50,

1896 0x34a74, 0x34c24,

1897 0x34d08, 0x34d14,

1898 0x34d1c, 0x34d20,

1899 0x34d3c, 0x34d50,

1900 0x35200, 0x3520c,

1901 0x35220, 0x35220,

1902 0x35240, 0x35240,

1903 0x35600, 0x35600,

1904 0x35608, 0x3560c,

1905 0x35a00, 0x35a1c,

1906 0x35e04, 0x35e20,

1907 0x35e38, 0x35e3c,

1908 0x35e80, 0x35e80,

1909 0x35e88, 0x35ea8,

1910 0x35eb0, 0x35eb4,

1911 0x35ec8, 0x35ed4,

1912 0x35fb8, 0x36004,

1913 0x36208, 0x3623c,

1914 0x36600, 0x36630,

1915 0x36a00, 0x36abc,

1916 0x36b00, 0x36b70,

1917 0x37000, 0x37048,

1918 0x37060, 0x3709c,

1919 0x370f0, 0x37148,

1920 0x37160, 0x3719c,

1921 0x371f0, 0x372e4,

1922 0x372f8, 0x373e4,

1923 0x373f8, 0x37448,

1924 0x37460, 0x3749c,

1925 0x374f0, 0x37548,

1926 0x37560, 0x3759c,

1927 0x375f0, 0x376e4,

1928 0x376f8, 0x377e4,

1929 0x377f8, 0x377fc,

1930 0x37814, 0x37814,

1931 0x3782c, 0x3782c,

1932 0x37880, 0x3788c,

1933 0x378e8, 0x378ec,

1934 0x37900, 0x37948,

1935 0x37960, 0x3799c,

1936 0x379f0, 0x37ae4,

1937 0x37af8, 0x37b10,

1938 0x37b28, 0x37b28,

1939 0x37b3c, 0x37b50,

1940 0x37bf0, 0x37c10,

1941 0x37c28, 0x37c28,

1942 0x37c3c, 0x37c50,

1943 0x37cf0, 0x37cfc,

1944 0x38000, 0x38030,

1945 0x38100, 0x38144,

1946 0x38190, 0x381d0,

1947 0x38200, 0x38318,

1948 0x38400, 0x3852c,

1949 0x38540, 0x3861c,

1950 0x38800, 0x38834,

1951 0x388c0, 0x38908,

1952 0x38910, 0x389ac,

1953 0x38a00, 0x38a04,

1954 0x38a0c, 0x38a2c,

1955 0x38a44, 0x38a50,

1956 0x38a74, 0x38c24,

1957 0x38d08, 0x38d14,

1958 0x38d1c, 0x38d20,

1959 0x38d3c, 0x38d50,

1960 0x39200, 0x3920c,

1961 0x39220, 0x39220,

1962 0x39240, 0x39240,

1963 0x39600, 0x39600,

1964 0x39608, 0x3960c,

1965 0x39a00, 0x39a1c,

1966 0x39e04, 0x39e20,

1967 0x39e38, 0x39e3c,

1968 0x39e80, 0x39e80,

1969 0x39e88, 0x39ea8,

1970 0x39eb0, 0x39eb4,

1971 0x39ec8, 0x39ed4,

1972 0x39fb8, 0x3a004,

1973 0x3a208, 0x3a23c,

1974 0x3a600, 0x3a630,

1975 0x3aa00, 0x3aabc,

1976 0x3ab00, 0x3ab70,

1977 0x3b000, 0x3b048,

1978 0x3b060, 0x3b09c,

1979 0x3b0f0, 0x3b148,

1980 0x3b160, 0x3b19c,

1981 0x3b1f0, 0x3b2e4,

1982 0x3b2f8, 0x3b3e4,

1983 0x3b3f8, 0x3b448,

1984 0x3b460, 0x3b49c,

1985 0x3b4f0, 0x3b548,

1986 0x3b560, 0x3b59c,

1987 0x3b5f0, 0x3b6e4,

1988 0x3b6f8, 0x3b7e4,

1989 0x3b7f8, 0x3b7fc,

1990 0x3b814, 0x3b814,

1991 0x3b82c, 0x3b82c,

1992 0x3b880, 0x3b88c,

1993 0x3b8e8, 0x3b8ec,

1994 0x3b900, 0x3b948,

1995 0x3b960, 0x3b99c,

1996 0x3b9f0, 0x3bae4,

1997 0x3baf8, 0x3bb10,

1998 0x3bb28, 0x3bb28,

1999 0x3bb3c, 0x3bb50,

2000 0x3bbf0, 0x3bc10,

2001 0x3bc28, 0x3bc28,

2002 0x3bc3c, 0x3bc50,

2003 0x3bcf0, 0x3bcfc,

2004 0x3c000, 0x3c030,

2005 0x3c100, 0x3c144,

2006 0x3c190, 0x3c1d0,

2007 0x3c200, 0x3c318,

2008 0x3c400, 0x3c52c,

2009 0x3c540, 0x3c61c,

2010 0x3c800, 0x3c834,

2011 0x3c8c0, 0x3c908,

2012 0x3c910, 0x3c9ac,

2013 0x3ca00, 0x3ca04,

2014 0x3ca0c, 0x3ca2c,

2015 0x3ca44, 0x3ca50,

2016 0x3ca74, 0x3cc24,

2017 0x3cd08, 0x3cd14,

2018 0x3cd1c, 0x3cd20,

2019 0x3cd3c, 0x3cd50,

2020 0x3d200, 0x3d20c,

2021 0x3d220, 0x3d220,

2022 0x3d240, 0x3d240,

2023 0x3d600, 0x3d600,

2024 0x3d608, 0x3d60c,

2025 0x3da00, 0x3da1c,

2026 0x3de04, 0x3de20,

2027 0x3de38, 0x3de3c,

2028 0x3de80, 0x3de80,

2029 0x3de88, 0x3dea8,

2030 0x3deb0, 0x3deb4,

2031 0x3dec8, 0x3ded4,

2032 0x3dfb8, 0x3e004,

2033 0x3e208, 0x3e23c,

2034 0x3e600, 0x3e630,

2035 0x3ea00, 0x3eabc,

2036 0x3eb00, 0x3eb70,

2037 0x3f000, 0x3f048,

2038 0x3f060, 0x3f09c,

2039 0x3f0f0, 0x3f148,

2040 0x3f160, 0x3f19c,

2041 0x3f1f0, 0x3f2e4,

2042 0x3f2f8, 0x3f3e4,

2043 0x3f3f8, 0x3f448,

2044 0x3f460, 0x3f49c,

2045 0x3f4f0, 0x3f548,

2046 0x3f560, 0x3f59c,

2047 0x3f5f0, 0x3f6e4,

2048 0x3f6f8, 0x3f7e4,

2049 0x3f7f8, 0x3f7fc,

2050 0x3f814, 0x3f814,

2051 0x3f82c, 0x3f82c,

2052 0x3f880, 0x3f88c,

2053 0x3f8e8, 0x3f8ec,

2054 0x3f900, 0x3f948,

2055 0x3f960, 0x3f99c,

2056 0x3f9f0, 0x3fae4,

2057 0x3faf8, 0x3fb10,

2058 0x3fb28, 0x3fb28,

2059 0x3fb3c, 0x3fb50,

2060 0x3fbf0, 0x3fc10,

2061 0x3fc28, 0x3fc28,

2062 0x3fc3c, 0x3fc50,

2063 0x3fcf0, 0x3fcfc,

2064 0x40000, 0x4000c,

2065 0x40040, 0x40068,

2066 0x40080, 0x40144,

2067 0x40180, 0x4018c,

2068 0x40200, 0x40298,

2069 0x402ac, 0x4033c,

2070 0x403f8, 0x403fc,

2071 0x41304, 0x413c4,

2072 0x41400, 0x4141c,

2073 0x41480, 0x414d0,

2074 0x44000, 0x44078,

2075 0x440c0, 0x44278,

2076 0x442c0, 0x44478,

2077 0x444c0, 0x44678,

2078 0x446c0, 0x44878,

2079 0x448c0, 0x449fc,

2080 0x45000, 0x45068,

2081 0x45080, 0x45084,

2082 0x450a0, 0x450b0,

2083 0x45200, 0x45268,

2084 0x45280, 0x45284,

2085 0x452a0, 0x452b0,

2086 0x460c0, 0x460e4,

2087 0x47000, 0x4708c,

2088 0x47200, 0x47250,

2089 0x47400, 0x47420,

2090 0x47600, 0x47618,

2091 0x47800, 0x47814,

2092 0x48000, 0x4800c,

2093 0x48040, 0x48068,

2094 0x48080, 0x48144,

2095 0x48180, 0x4818c,

2096 0x48200, 0x48298,

2097 0x482ac, 0x4833c,

2098 0x483f8, 0x483fc,

2099 0x49304, 0x493c4,

2100 0x49400, 0x4941c,

2101 0x49480, 0x494d0,

2102 0x4c000, 0x4c078,

2103 0x4c0c0, 0x4c278,

2104 0x4c2c0, 0x4c478,

2105 0x4c4c0, 0x4c678,

2106 0x4c6c0, 0x4c878,

2107 0x4c8c0, 0x4c9fc,

2108 0x4d000, 0x4d068,

2109 0x4d080, 0x4d084,

2110 0x4d0a0, 0x4d0b0,

2111 0x4d200, 0x4d268,

2112 0x4d280, 0x4d284,

2113 0x4d2a0, 0x4d2b0,

2114 0x4e0c0, 0x4e0e4,

2115 0x4f000, 0x4f08c,

2116 0x4f200, 0x4f250,

2117 0x4f400, 0x4f420,

2118 0x4f600, 0x4f618,

2119 0x4f800, 0x4f814,

2120 0x50000, 0x500cc,

2121 0x50400, 0x50400,

2122 0x50800, 0x508cc,

2123 0x50c00, 0x50c00,

2124 0x51000, 0x5101c,

2125 0x51300, 0x51308,

2126 };

2127

2128 int i;

2129 struct adapter *ap = netdev2adap(dev);

2130 static const unsigned int *reg_ranges;

2131 int arr_size = 0, buf_size = 0;

2132

2133 if (is_t4(ap->params.chip)) {

2134 reg_ranges = &t4_reg_ranges[0];

2135 arr_size = ARRAY_SIZE(t4_reg_ranges);

2136 buf_size = T4_REGMAP_SIZE;

2137 } else {

2138 reg_ranges = &t5_reg_ranges[0];

2139 arr_size = ARRAY_SIZE(t5_reg_ranges);

2140 buf_size = T5_REGMAP_SIZE;

2141 }

2142

2143 regs->version = mk_adap_vers(ap);

2144

2145 memset(buf, 0, buf_size);

2146 for (i = 0; i < arr_size; i += 2)

2147 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);

2148 }

2149

2150 static int restart_autoneg(struct net_device *dev)

2151 {

2152 struct port_info *p = netdev_priv(dev);

2153

2154 if (!netif_running(dev))

2155 return -EAGAIN;

2156 if (p->link_cfg.autoneg != AUTONEG_ENABLE)

2157 return -EINVAL;

2158 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);

2159 return 0;

2160 }

2161

2162 static int identify_port(struct net_device *dev,

2163 enum ethtool_phys_id_state state)

2164 {

2165 unsigned int val;

2166 struct adapter *adap = netdev2adap(dev);

2167

2168 if (state == ETHTOOL_ID_ACTIVE)

2169 val = 0xffff;

2170 else if (state == ETHTOOL_ID_INACTIVE)

2171 val = 0;

2172 else

2173 return -EINVAL;

2174

2175 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);

2176 }

2177

2178 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)

2179 {

2180 unsigned int v = 0;

2181

2182 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||

2183 type == FW_PORT_TYPE_BT_XAUI) {

2184 v |= SUPPORTED_TP;

2185 if (caps & FW_PORT_CAP_SPEED_100M)

2186 v |= SUPPORTED_100baseT_Full;

2187 if (caps & FW_PORT_CAP_SPEED_1G)

2188 v |= SUPPORTED_1000baseT_Full;

2189 if (caps & FW_PORT_CAP_SPEED_10G)

2190 v |= SUPPORTED_10000baseT_Full;

2191 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {

2192 v |= SUPPORTED_Backplane;

2193 if (caps & FW_PORT_CAP_SPEED_1G)

2194 v |= SUPPORTED_1000baseKX_Full;

2195 if (caps & FW_PORT_CAP_SPEED_10G)

2196 v |= SUPPORTED_10000baseKX4_Full;

2197 } else if (type == FW_PORT_TYPE_KR)

2198 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;

2199 else if (type == FW_PORT_TYPE_BP_AP)

2200 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |

2201 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;

2202 else if (type == FW_PORT_TYPE_BP4_AP)

2203 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |

2204 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |

2205 SUPPORTED_10000baseKX4_Full;

2206 else if (type == FW_PORT_TYPE_FIBER_XFI ||

2207 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)

2208 v |= SUPPORTED_FIBRE;

2209 else if (type == FW_PORT_TYPE_BP40_BA)

2210 v |= SUPPORTED_40000baseSR4_Full;

2211

2212 if (caps & FW_PORT_CAP_ANEG)

2213 v |= SUPPORTED_Autoneg;

2214 return v;

2215 }

2216

2217 static unsigned int to_fw_linkcaps(unsigned int caps)

2218 {

2219 unsigned int v = 0;

2220

2221 if (caps & ADVERTISED_100baseT_Full)

2222 v |= FW_PORT_CAP_SPEED_100M;

2223 if (caps & ADVERTISED_1000baseT_Full)

2224 v |= FW_PORT_CAP_SPEED_1G;

2225 if (caps & ADVERTISED_10000baseT_Full)

2226 v |= FW_PORT_CAP_SPEED_10G;

2227 if (caps & ADVERTISED_40000baseSR4_Full)

2228 v |= FW_PORT_CAP_SPEED_40G;

2229 return v;

2230 }

2231

2232 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)

2233 {

2234 const struct port_info *p = netdev_priv(dev);

2235

2236 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||

2237 p->port_type == FW_PORT_TYPE_BT_XFI ||

2238 p->port_type == FW_PORT_TYPE_BT_XAUI)

2239 cmd->port = PORT_TP;

2240 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||

2241 p->port_type == FW_PORT_TYPE_FIBER_XAUI)

2242 cmd->port = PORT_FIBRE;

2243 else if (p->port_type == FW_PORT_TYPE_SFP) {

2244 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||

2245 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)

2246 cmd->port = PORT_DA;

2247 else

2248 cmd->port = PORT_FIBRE;

2249 } else

2250 cmd->port = PORT_OTHER;

2251

2252 if (p->mdio_addr >= 0) {

2253 cmd->phy_address = p->mdio_addr;

2254 cmd->transceiver = XCVR_EXTERNAL;

2255 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?

2256 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;

2257 } else {

2258 cmd->phy_address = 0; /* not really, but no better option */

2259 cmd->transceiver = XCVR_INTERNAL;

2260 cmd->mdio_support = 0;

2261 }

2262

2263 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);

2264 cmd->advertising = from_fw_linkcaps(p->port_type,

2265 p->link_cfg.advertising);

2266 ethtool_cmd_speed_set(cmd,

2267 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);

2268 cmd->duplex = DUPLEX_FULL;

2269 cmd->autoneg = p->link_cfg.autoneg;

2270 cmd->maxtxpkt = 0;

2271 cmd->maxrxpkt = 0;

2272 return 0;

2273 }

2274

2275 static unsigned int speed_to_caps(int speed)

2276 {

2277 if (speed == 100)

2278 return FW_PORT_CAP_SPEED_100M;

2279 if (speed == 1000)

2280 return FW_PORT_CAP_SPEED_1G;

2281 if (speed == 10000)

2282 return FW_PORT_CAP_SPEED_10G;

2283 if (speed == 40000)

2284 return FW_PORT_CAP_SPEED_40G;

2285 return 0;

2286 }

2287

2288 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)

2289 {

2290 unsigned int cap;

2291 struct port_info *p = netdev_priv(dev);

2292 struct link_config *lc = &p->link_cfg;

2293 u32 speed = ethtool_cmd_speed(cmd);

2294

2295 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */

2296 return -EINVAL;

2297

2298 if (!(lc->supported & FW_PORT_CAP_ANEG)) {

2299 /*

2300 * PHY offers a single speed. See if that's what's

2301 * being requested.

2302 */

2303 if (cmd->autoneg == AUTONEG_DISABLE &&

2304 (lc->supported & speed_to_caps(speed)))

2305 return 0;

2306 return -EINVAL;

2307 }

2308

2309 if (cmd->autoneg == AUTONEG_DISABLE) {

2310 cap = speed_to_caps(speed);

2311

2312 if (!(lc->supported & cap) ||

2313 (speed == 1000) ||

2314 (speed == 10000) ||

2315 (speed == 40000))

2316 return -EINVAL;

2317 lc->requested_speed = cap;

2318 lc->advertising = 0;

2319 } else {

2320 cap = to_fw_linkcaps(cmd->advertising);

2321 if (!(lc->supported & cap))

2322 return -EINVAL;

2323 lc->requested_speed = 0;

2324 lc->advertising = cap | FW_PORT_CAP_ANEG;

2325 }

2326 lc->autoneg = cmd->autoneg;

2327

2328 if (netif_running(dev))

2329 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,

2330 lc);

2331 return 0;

2332 }

2333

2334 static void get_pauseparam(struct net_device *dev,

2335 struct ethtool_pauseparam *epause)

2336 {

2337 struct port_info *p = netdev_priv(dev);

2338

2339 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;

2340 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;

2341 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;

2342 }

2343

2344 static int set_pauseparam(struct net_device *dev,

2345 struct ethtool_pauseparam *epause)

2346 {

2347 struct port_info *p = netdev_priv(dev);

2348 struct link_config *lc = &p->link_cfg;

2349

2350 if (epause->autoneg == AUTONEG_DISABLE)

2351 lc->requested_fc = 0;

2352 else if (lc->supported & FW_PORT_CAP_ANEG)

2353 lc->requested_fc = PAUSE_AUTONEG;

2354 else

2355 return -EINVAL;

2356

2357 if (epause->rx_pause)

2358 lc->requested_fc |= PAUSE_RX;

2359 if (epause->tx_pause)

2360 lc->requested_fc |= PAUSE_TX;

2361 if (netif_running(dev))

2362 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,

2363 lc);

2364 return 0;

2365 }

2366

2367 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)

2368 {

2369 const struct port_info *pi = netdev_priv(dev);

2370 const struct sge *s = &pi->adapter->sge;

2371

2372 e->rx_max_pending = MAX_RX_BUFFERS;

2373 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;

2374 e->rx_jumbo_max_pending = 0;

2375 e->tx_max_pending = MAX_TXQ_ENTRIES;

2376

2377 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;

2378 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;

2379 e->rx_jumbo_pending = 0;

2380 e->tx_pending = s->ethtxq[pi->first_qset].q.size;

2381 }

2382

2383 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)

2384 {

2385 int i;

2386 const struct port_info *pi = netdev_priv(dev);

2387 struct adapter *adapter = pi->adapter;

2388 struct sge *s = &adapter->sge;

2389

2390 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||

2391 e->tx_pending > MAX_TXQ_ENTRIES ||

2392 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||

2393 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||

2394 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)

2395 return -EINVAL;

2396

2397 if (adapter->flags & FULL_INIT_DONE)

2398 return -EBUSY;

2399

2400 for (i = 0; i < pi->nqsets; ++i) {

2401 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;

2402 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;

2403 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;

2404 }

2405 return 0;

2406 }

2407

2408 static int closest_timer(const struct sge *s, int time)

2409 {

2410 int i, delta, match = 0, min_delta = INT_MAX;

2411

2412 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {

2413 delta = time - s->timer_val[i];

2414 if (delta < 0)

2415 delta = -delta;

2416 if (delta < min_delta) {

2417 min_delta = delta;

2418 match = i;

2419 }

2420 }

2421 return match;

2422 }

2423

2424 static int closest_thres(const struct sge *s, int thres)

2425 {

2426 int i, delta, match = 0, min_delta = INT_MAX;

2427

2428 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {

2429 delta = thres - s->counter_val[i];

2430 if (delta < 0)

2431 delta = -delta;

2432 if (delta < min_delta) {

2433 min_delta = delta;

2434 match = i;

2435 }

2436 }

2437 return match;

2438 }

2439

2440 /*

2441 * Return a queue's interrupt hold-off time in us. 0 means no timer.

2442 */

2443 static unsigned int qtimer_val(const struct adapter *adap,

2444 const struct sge_rspq *q)

2445 {

2446 unsigned int idx = q->intr_params >> 1;

2447

2448 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;

2449 }

2450

2451 /**

2452 * set_rxq_intr_params - set a queue's interrupt holdoff parameters

2453 * @adap: the adapter

2454 * @q: the Rx queue

2455 * @us: the hold-off time in us, or 0 to disable timer

2456 * @cnt: the hold-off packet count, or 0 to disable counter

2457 *

2458 * Sets an Rx queue's interrupt hold-off time and packet count. At least

2459 * one of the two needs to be enabled for the queue to generate interrupts.

2460 */

2461 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,

2462 unsigned int us, unsigned int cnt)

2463 {

2464 if ((us | cnt) == 0)

2465 cnt = 1;

2466

2467 if (cnt) {

2468 int err;

2469 u32 v, new_idx;

2470

2471 new_idx = closest_thres(&adap->sge, cnt);

2472 if (q->desc && q->pktcnt_idx != new_idx) {

2473 /* the queue has already been created, update it */

2474 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |

2475 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |

2476 FW_PARAMS_PARAM_YZ(q->cntxt_id);

2477 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,

2478 &new_idx);

2479 if (err)

2480 return err;

2481 }

2482 q->pktcnt_idx = new_idx;

2483 }

2484

2485 us = us == 0 ? 6 : closest_timer(&adap->sge, us);

2486 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);

2487 return 0;

2488 }

2489

2490 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)

2491 {

2492 const struct port_info *pi = netdev_priv(dev);

2493 struct adapter *adap = pi->adapter;

2494 struct sge_rspq *q;

2495 int i;

2496 int r = 0;

2497

2498 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) {

2499 q = &adap->sge.ethrxq[i].rspq;

2500 r = set_rxq_intr_params(adap, q, c->rx_coalesce_usecs,

2501 c->rx_max_coalesced_frames);

2502 if (r) {

2503 dev_err(&dev->dev, "failed to set coalesce %d\n", r);

2504 break;

2505 }

2506 }

2507 return r;

2508 }

2509

2510 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)

2511 {

2512 const struct port_info *pi = netdev_priv(dev);

2513 const struct adapter *adap = pi->adapter;

2514 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;

2515

2516 c->rx_coalesce_usecs = qtimer_val(adap, rq);

2517 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?

2518 adap->sge.counter_val[rq->pktcnt_idx] : 0;

2519 return 0;

2520 }

2521

2522 /**

2523 * eeprom_ptov - translate a physical EEPROM address to virtual

2524 * @phys_addr: the physical EEPROM address

2525 * @fn: the PCI function number

2526 * @sz: size of function-specific area

2527 *

2528 * Translate a physical EEPROM address to virtual. The first 1K is

2529 * accessed through virtual addresses starting at 31K, the rest is

2530 * accessed through virtual addresses starting at 0.

2531 *

2532 * The mapping is as follows:

2533 * [0..1K) -> [31K..32K)

2534 * [1K..1K+A) -> [31K-A..31K)

2535 * [1K+A..ES) -> [0..ES-A-1K)

2536 *

2537 * where A = @fn * @sz, and ES = EEPROM size.

2538 */

2539 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)

2540 {

2541 fn *= sz;

2542 if (phys_addr < 1024)

2543 return phys_addr + (31 << 10);

2544 if (phys_addr < 1024 + fn)

2545 return 31744 - fn + phys_addr - 1024;

2546 if (phys_addr < EEPROMSIZE)

2547 return phys_addr - 1024 - fn;

2548 return -EINVAL;

2549 }

2550

2551 /*

2552 * The next two routines implement eeprom read/write from physical addresses.

2553 */

2554 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)

2555 {

2556 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);

2557

2558 if (vaddr >= 0)

2559 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);

2560 return vaddr < 0 ? vaddr : 0;

2561 }

2562

2563 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)

2564 {

2565 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);

2566

2567 if (vaddr >= 0)

2568 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);

2569 return vaddr < 0 ? vaddr : 0;

2570 }

2571

2572 #define EEPROM_MAGIC 0x38E2F10C

2573

2574 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,

2575 u8 *data)

2576 {

2577 int i, err = 0;

2578 struct adapter *adapter = netdev2adap(dev);

2579

2580 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);

2581 if (!buf)

2582 return -ENOMEM;

2583

2584 e->magic = EEPROM_MAGIC;

2585 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)

2586 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);

2587

2588 if (!err)

2589 memcpy(data, buf + e->offset, e->len);

2590 kfree(buf);

2591 return err;

2592 }

2593

2594 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,

2595 u8 *data)

2596 {

2597 u8 *buf;

2598 int err = 0;

2599 u32 aligned_offset, aligned_len, *p;

2600 struct adapter *adapter = netdev2adap(dev);

2601

2602 if (eeprom->magic != EEPROM_MAGIC)

2603 return -EINVAL;

2604

2605 aligned_offset = eeprom->offset & ~3;

2606 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;

2607

2608 if (adapter->fn > 0) {

2609 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;

2610

2611 if (aligned_offset < start ||

2612 aligned_offset + aligned_len > start + EEPROMPFSIZE)

2613 return -EPERM;

2614 }

2615

2616 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {

2617 /*

2618 * RMW possibly needed for first or last words.

2619 */

2620 buf = kmalloc(aligned_len, GFP_KERNEL);

2621 if (!buf)

2622 return -ENOMEM;

2623 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);

2624 if (!err && aligned_len > 4)

2625 err = eeprom_rd_phys(adapter,

2626 aligned_offset + aligned_len - 4,

2627 (u32 *)&buf[aligned_len - 4]);

2628 if (err)

2629 goto out;

2630 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);

2631 } else

2632 buf = data;

2633

2634 err = t4_seeprom_wp(adapter, false);

2635 if (err)

2636 goto out;

2637

2638 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {

2639 err = eeprom_wr_phys(adapter, aligned_offset, *p);

2640 aligned_offset += 4;

2641 }

2642

2643 if (!err)

2644 err = t4_seeprom_wp(adapter, true);

2645 out:

2646 if (buf != data)

2647 kfree(buf);

2648 return err;

2649 }

2650

2651 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)

2652 {

2653 int ret;

2654 const struct firmware *fw;

2655 struct adapter *adap = netdev2adap(netdev);

2656

2657 ef->data[sizeof(ef->data) - 1] = '\0';

2658 ret = request_firmware(&fw, ef->data, adap->pdev_dev);

2659 if (ret < 0)

2660 return ret;

2661

2662 ret = t4_load_fw(adap, fw->data, fw->size);

2663 release_firmware(fw);

2664 if (!ret)

2665 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);

2666 return ret;

2667 }

2668

2669 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)

2670 #define BCAST_CRC 0xa0ccc1a6

2671

2672 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)

2673 {

2674 wol->supported = WAKE_BCAST | WAKE_MAGIC;

2675 wol->wolopts = netdev2adap(dev)->wol;

2676 memset(&wol->sopass, 0, sizeof(wol->sopass));

2677 }

2678

2679 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)

2680 {

2681 int err = 0;

2682 struct port_info *pi = netdev_priv(dev);

2683

2684 if (wol->wolopts & ~WOL_SUPPORTED)

2685 return -EINVAL;

2686 t4_wol_magic_enable(pi->adapter, pi->tx_chan,

2687 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);

2688 if (wol->wolopts & WAKE_BCAST) {

2689 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,

2690 ~0ULL, 0, false);

2691 if (!err)

2692 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,

2693 ~6ULL, ~0ULL, BCAST_CRC, true);

2694 } else

2695 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);

2696 return err;

2697 }

2698

2699 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)

2700 {

2701 const struct port_info *pi = netdev_priv(dev);

2702 netdev_features_t changed = dev->features ^ features;

2703 int err;

2704

2705 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))

2706 return 0;

2707

2708 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,

2709 -1, -1, -1,

2710 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);

2711 if (unlikely(err))

2712 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;

2713 return err;

2714 }

2715

2716 static u32 get_rss_table_size(struct net_device *dev)

2717 {

2718 const struct port_info *pi = netdev_priv(dev);

2719

2720 return pi->rss_size;

2721 }

2722

2723 static int get_rss_table(struct net_device *dev, u32 *p)

2724 {

2725 const struct port_info *pi = netdev_priv(dev);

2726 unsigned int n = pi->rss_size;

2727

2728 while (n--)

2729 p[n] = pi->rss[n];

2730 return 0;

2731 }

2732

2733 static int set_rss_table(struct net_device *dev, const u32 *p)

2734 {

2735 unsigned int i;

2736 struct port_info *pi = netdev_priv(dev);

2737

2738 for (i = 0; i < pi->rss_size; i++)

2739 pi->rss[i] = p[i];

2740 if (pi->adapter->flags & FULL_INIT_DONE)

2741 return write_rss(pi, pi->rss);

2742 return 0;

2743 }

2744

2745 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,

2746 u32 *rules)

2747 {

2748 const struct port_info *pi = netdev_priv(dev);

2749

2750 switch (info->cmd) {

2751 case ETHTOOL_GRXFH: {

2752 unsigned int v = pi->rss_mode;

2753

2754 info->data = 0;

2755 switch (info->flow_type) {

2756 case TCP_V4_FLOW:

2757 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)

2758 info->data = RXH_IP_SRC | RXH_IP_DST |

2759 RXH_L4_B_0_1 | RXH_L4_B_2_3;

2760 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)

2761 info->data = RXH_IP_SRC | RXH_IP_DST;

2762 break;

2763 case UDP_V4_FLOW:

2764 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&

2765 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))

2766 info->data = RXH_IP_SRC | RXH_IP_DST |

2767 RXH_L4_B_0_1 | RXH_L4_B_2_3;

2768 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)

2769 info->data = RXH_IP_SRC | RXH_IP_DST;

2770 break;

2771 case SCTP_V4_FLOW:

2772 case AH_ESP_V4_FLOW:

2773 case IPV4_FLOW:

2774 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)

2775 info->data = RXH_IP_SRC | RXH_IP_DST;

2776 break;

2777 case TCP_V6_FLOW:

2778 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)

2779 info->data = RXH_IP_SRC | RXH_IP_DST |

2780 RXH_L4_B_0_1 | RXH_L4_B_2_3;

2781 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)

2782 info->data = RXH_IP_SRC | RXH_IP_DST;

2783 break;

2784 case UDP_V6_FLOW:

2785 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&

2786 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))

2787 info->data = RXH_IP_SRC | RXH_IP_DST |

2788 RXH_L4_B_0_1 | RXH_L4_B_2_3;

2789 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)

2790 info->data = RXH_IP_SRC | RXH_IP_DST;

2791 break;

2792 case SCTP_V6_FLOW:

2793 case AH_ESP_V6_FLOW:

2794 case IPV6_FLOW:

2795 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)

2796 info->data = RXH_IP_SRC | RXH_IP_DST;

2797 break;

2798 }

2799 return 0;

2800 }

2801 case ETHTOOL_GRXRINGS:

2802 info->data = pi->nqsets;

2803 return 0;

2804 }

2805 return -EOPNOTSUPP;

2806 }

2807

2808 static const struct ethtool_ops cxgb_ethtool_ops = {

2809 .get_settings = get_settings,

2810 .set_settings = set_settings,

2811 .get_drvinfo = get_drvinfo,

2812 .get_msglevel = get_msglevel,

2813 .set_msglevel = set_msglevel,

2814 .get_ringparam = get_sge_param,

2815 .set_ringparam = set_sge_param,

2816 .get_coalesce = get_coalesce,

2817 .set_coalesce = set_coalesce,

2818 .get_eeprom_len = get_eeprom_len,

2819 .get_eeprom = get_eeprom,

2820 .set_eeprom = set_eeprom,

2821 .get_pauseparam = get_pauseparam,

2822 .set_pauseparam = set_pauseparam,

2823 .get_link = ethtool_op_get_link,

2824 .get_strings = get_strings,

2825 .set_phys_id = identify_port,

2826 .nway_reset = restart_autoneg,

2827 .get_sset_count = get_sset_count,

2828 .get_ethtool_stats = get_stats,

2829 .get_regs_len = get_regs_len,

2830 .get_regs = get_regs,

2831 .get_wol = get_wol,

2832 .set_wol = set_wol,

2833 .get_rxnfc = get_rxnfc,

2834 .get_rxfh_indir_size = get_rss_table_size,

2835 .get_rxfh_indir = get_rss_table,

2836 .set_rxfh_indir = set_rss_table,

2837 .flash_device = set_flash,

2838 };

2839

2840 /*

2841 * debugfs support

2842 */

2843 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,

2844 loff_t *ppos)

2845 {

2846 loff_t pos = *ppos;

2847 loff_t avail = file_inode(file)->i_size;

2848 unsigned int mem = (uintptr_t)file->private_data & 3;

2849 struct adapter *adap = file->private_data - mem;

2850

2851 if (pos < 0)

2852 return -EINVAL;

2853 if (pos >= avail)

2854 return 0;

2855 if (count > avail - pos)

2856 count = avail - pos;

2857

2858 while (count) {

2859 size_t len;

2860 int ret, ofst;

2861 __be32 data[16];

2862

2863 if ((mem == MEM_MC) || (mem == MEM_MC1))

2864 ret = t4_mc_read(adap, mem % MEM_MC, pos, data, NULL);

2865 else

2866 ret = t4_edc_read(adap, mem, pos, data, NULL);

2867 if (ret)

2868 return ret;

2869

2870 ofst = pos % sizeof(data);

2871 len = min(count, sizeof(data) - ofst);

2872 if (copy_to_user(buf, (u8 *)data + ofst, len))

2873 return -EFAULT;

2874

2875 buf += len;

2876 pos += len;

2877 count -= len;

2878 }

2879 count = pos - *ppos;

2880 *ppos = pos;

2881 return count;

2882 }

2883

2884 static const struct file_operations mem_debugfs_fops = {

2885 .owner = THIS_MODULE,

2886 .open = simple_open,

2887 .read = mem_read,

2888 .llseek = default_llseek,

2889 };

2890

2891 static void add_debugfs_mem(struct adapter *adap, const char *name,

2892 unsigned int idx, unsigned int size_mb)

2893 {

2894 struct dentry *de;

2895

2896 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,

2897 (void *)adap + idx, &mem_debugfs_fops);

2898 if (de && de->d_inode)

2899 de->d_inode->i_size = size_mb << 20;

2900 }

2901

2902 static int setup_debugfs(struct adapter *adap)

2903 {

2904 int i;

2905 u32 size;

2906

2907 if (IS_ERR_OR_NULL(adap->debugfs_root))

2908 return -1;

2909

2910 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);

2911 if (i & EDRAM0_ENABLE) {

2912 size = t4_read_reg(adap, MA_EDRAM0_BAR);

2913 add_debugfs_mem(adap, "edc0", MEM_EDC0, EDRAM_SIZE_GET(size));

2914 }

2915 if (i & EDRAM1_ENABLE) {

2916 size = t4_read_reg(adap, MA_EDRAM1_BAR);

2917 add_debugfs_mem(adap, "edc1", MEM_EDC1, EDRAM_SIZE_GET(size));

2918 }

2919 if (is_t4(adap->params.chip)) {

2920 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);

2921 if (i & EXT_MEM_ENABLE)

2922 add_debugfs_mem(adap, "mc", MEM_MC,

2923 EXT_MEM_SIZE_GET(size));

2924 } else {

2925 if (i & EXT_MEM_ENABLE) {

2926 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);

2927 add_debugfs_mem(adap, "mc0", MEM_MC0,

2928 EXT_MEM_SIZE_GET(size));

2929 }

2930 if (i & EXT_MEM1_ENABLE) {

2931 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR);

2932 add_debugfs_mem(adap, "mc1", MEM_MC1,

2933 EXT_MEM_SIZE_GET(size));

2934 }

2935 }

2936 if (adap->l2t)

2937 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,

2938 &t4_l2t_fops);

2939 return 0;

2940 }

2941

2942 /*

2943 * upper-layer driver support

2944 */

2945

2946 /*

2947 * Allocate an active-open TID and set it to the supplied value.

2948 */

2949 int cxgb4_alloc_atid(struct tid_info *t, void *data)

2950 {

2951 int atid = -1;

2952

2953 spin_lock_bh(&t->atid_lock);

2954 if (t->afree) {

2955 union aopen_entry *p = t->afree;

2956

2957 atid = (p - t->atid_tab) + t->atid_base;

2958 t->afree = p->next;

2959 p->data = data;

2960 t->atids_in_use++;

2961 }

2962 spin_unlock_bh(&t->atid_lock);

2963 return atid;

2964 }

2965 EXPORT_SYMBOL(cxgb4_alloc_atid);

2966

2967 /*

2968 * Release an active-open TID.

2969 */

2970 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)

2971 {

2972 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];

2973

2974 spin_lock_bh(&t->atid_lock);

2975 p->next = t->afree;

2976 t->afree = p;

2977 t->atids_in_use--;

2978 spin_unlock_bh(&t->atid_lock);

2979 }

2980 EXPORT_SYMBOL(cxgb4_free_atid);

2981

2982 /*

2983 * Allocate a server TID and set it to the supplied value.

2984 */

2985 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)

2986 {

2987 int stid;

2988

2989 spin_lock_bh(&t->stid_lock);

2990 if (family == PF_INET) {

2991 stid = find_first_zero_bit(t->stid_bmap, t->nstids);

2992 if (stid < t->nstids)

2993 __set_bit(stid, t->stid_bmap);

2994 else

2995 stid = -1;

2996 } else {

2997 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);

2998 if (stid < 0)

2999 stid = -1;

3000 }

3001 if (stid >= 0) {

3002 t->stid_tab[stid].data = data;

3003 stid += t->stid_base;

3004 /* IPv6 requires max of 520 bits or 16 cells in TCAM

3005 * This is equivalent to 4 TIDs. With CLIP enabled it

3006 * needs 2 TIDs.

3007 */

3008 if (family == PF_INET)

3009 t->stids_in_use++;

3010 else

3011 t->stids_in_use += 4;

3012 }

3013 spin_unlock_bh(&t->stid_lock);

3014 return stid;

3015 }

3016 EXPORT_SYMBOL(cxgb4_alloc_stid);

3017

3018 /* Allocate a server filter TID and set it to the supplied value.

3019 */

3020 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)

3021 {

3022 int stid;

3023

3024 spin_lock_bh(&t->stid_lock);

3025 if (family == PF_INET) {

3026 stid = find_next_zero_bit(t->stid_bmap,

3027 t->nstids + t->nsftids, t->nstids);

3028 if (stid < (t->nstids + t->nsftids))

3029 __set_bit(stid, t->stid_bmap);

3030 else

3031 stid = -1;

3032 } else {

3033 stid = -1;

3034 }

3035 if (stid >= 0) {

3036 t->stid_tab[stid].data = data;

3037 stid -= t->nstids;

3038 stid += t->sftid_base;

3039 t->stids_in_use++;

3040 }

3041 spin_unlock_bh(&t->stid_lock);

3042 return stid;

3043 }

3044 EXPORT_SYMBOL(cxgb4_alloc_sftid);

3045

3046 /* Release a server TID.

3047 */

3048 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)

3049 {

3050 /* Is it a server filter TID? */

3051 if (t->nsftids && (stid >= t->sftid_base)) {

3052 stid -= t->sftid_base;

3053 stid += t->nstids;

3054 } else {

3055 stid -= t->stid_base;

3056 }

3057

3058 spin_lock_bh(&t->stid_lock);

3059 if (family == PF_INET)

3060 __clear_bit(stid, t->stid_bmap);

3061 else

3062 bitmap_release_region(t->stid_bmap, stid, 2);

3063 t->stid_tab[stid].data = NULL;

3064 if (family == PF_INET)

3065 t->stids_in_use--;

3066 else

3067 t->stids_in_use -= 4;

3068 spin_unlock_bh(&t->stid_lock);

3069 }

3070 EXPORT_SYMBOL(cxgb4_free_stid);

3071

3072 /*

3073 * Populate a TID_RELEASE WR. Caller must properly size the skb.

3074 */

3075 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,

3076 unsigned int tid)

3077 {

3078 struct cpl_tid_release *req;

3079

3080 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);

3081 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));

3082 INIT_TP_WR(req, tid);

3083 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));

3084 }

3085

3086 /*

3087 * Queue a TID release request and if necessary schedule a work queue to

3088 * process it.

3089 */

3090 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,

3091 unsigned int tid)

3092 {

3093 void **p = &t->tid_tab[tid];

3094 struct adapter *adap = container_of(t, struct adapter, tids);

3095

3096 spin_lock_bh(&adap->tid_release_lock);

3097 *p = adap->tid_release_head;

3098 /* Low 2 bits encode the Tx channel number */

3099 adap->tid_release_head = (void **)((uintptr_t)p | chan);

3100 if (!adap->tid_release_task_busy) {

3101 adap->tid_release_task_busy = true;

3102 queue_work(workq, &adap->tid_release_task);

3103 }

3104 spin_unlock_bh(&adap->tid_release_lock);

3105 }

3106

3107 /*

3108 * Process the list of pending TID release requests.

3109 */

3110 static void process_tid_release_list(struct work_struct *work)

3111 {

3112 struct sk_buff *skb;

3113 struct adapter *adap;

3114

3115 adap = container_of(work, struct adapter, tid_release_task);

3116

3117 spin_lock_bh(&adap->tid_release_lock);

3118 while (adap->tid_release_head) {

3119 void **p = adap->tid_release_head;

3120 unsigned int chan = (uintptr_t)p & 3;

3121 p = (void *)p - chan;

3122

3123 adap->tid_release_head = *p;

3124 *p = NULL;

3125 spin_unlock_bh(&adap->tid_release_lock);

3126

3127 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),

3128 GFP_KERNEL)))

3129 schedule_timeout_uninterruptible(1);

3130

3131 mk_tid_release(skb, chan, p - adap->tids.tid_tab);

3132 t4_ofld_send(adap, skb);

3133 spin_lock_bh(&adap->tid_release_lock);

3134 }

3135 adap->tid_release_task_busy = false;

3136 spin_unlock_bh(&adap->tid_release_lock);

3137 }

3138

3139 /*

3140 * Release a TID and inform HW. If we are unable to allocate the release

3141 * message we defer to a work queue.

3142 */

3143 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)

3144 {

3145 void *old;

3146 struct sk_buff *skb;

3147 struct adapter *adap = container_of(t, struct adapter, tids);

3148

3149 old = t->tid_tab[tid];

3150 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);

3151 if (likely(skb)) {

3152 t->tid_tab[tid] = NULL;

3153 mk_tid_release(skb, chan, tid);

3154 t4_ofld_send(adap, skb);

3155 } else

3156 cxgb4_queue_tid_release(t, chan, tid);

3157 if (old)

3158 atomic_dec(&t->tids_in_use);

3159 }

3160 EXPORT_SYMBOL(cxgb4_remove_tid);

3161

3162 /*

3163 * Allocate and initialize the TID tables. Returns 0 on success.

3164 */

3165 static int tid_init(struct tid_info *t)

3166 {

3167 size_t size;

3168 unsigned int stid_bmap_size;

3169 unsigned int natids = t->natids;

3170 struct adapter *adap = container_of(t, struct adapter, tids);

3171

3172 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);

3173 size = t->ntids * sizeof(*t->tid_tab) +

3174 natids * sizeof(*t->atid_tab) +

3175 t->nstids * sizeof(*t->stid_tab) +

3176 t->nsftids * sizeof(*t->stid_tab) +

3177 stid_bmap_size * sizeof(long) +

3178 t->nftids * sizeof(*t->ftid_tab) +

3179 t->nsftids * sizeof(*t->ftid_tab);

3180

3181 t->tid_tab = t4_alloc_mem(size);

3182 if (!t->tid_tab)

3183 return -ENOMEM;

3184

3185 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];

3186 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];

3187 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];

3188 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];

3189 spin_lock_init(&t->stid_lock);

3190 spin_lock_init(&t->atid_lock);

3191

3192 t->stids_in_use = 0;

3193 t->afree = NULL;

3194 t->atids_in_use = 0;

3195 atomic_set(&t->tids_in_use, 0);

3196

3197 /* Setup the free list for atid_tab and clear the stid bitmap. */

3198 if (natids) {

3199 while (--natids)

3200 t->atid_tab[natids - 1].next = &t->atid_tab[natids];

3201 t->afree = t->atid_tab;

3202 }

3203 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);

3204 /* Reserve stid 0 for T4/T5 adapters */

3205 if (!t->stid_base &&

3206 (is_t4(adap->params.chip) || is_t5(adap->params.chip)))

3207 __set_bit(0, t->stid_bmap);

3208

3209 return 0;

3210 }

3211

3212 static int cxgb4_clip_get(const struct net_device *dev,

3213 const struct in6_addr *lip)

3214 {

3215 struct adapter *adap;

3216 struct fw_clip_cmd c;

3217

3218 adap = netdev2adap(dev);

3219 memset(&c, 0, sizeof(c));

3220 c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |

3221 FW_CMD_REQUEST | FW_CMD_WRITE);

3222 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c));

3223 *(__be64 *)&c.ip_hi = *(__be64 *)(lip->s6_addr);

3224 *(__be64 *)&c.ip_lo = *(__be64 *)(lip->s6_addr + 8);

3225 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);

3226 }

3227

3228 static int cxgb4_clip_release(const struct net_device *dev,

3229 const struct in6_addr *lip)

3230 {

3231 struct adapter *adap;

3232 struct fw_clip_cmd c;

3233

3234 adap = netdev2adap(dev);

3235 memset(&c, 0, sizeof(c));

3236 c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |

3237 FW_CMD_REQUEST | FW_CMD_READ);

3238 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_FREE | FW_LEN16(c));

3239 *(__be64 *)&c.ip_hi = *(__be64 *)(lip->s6_addr);

3240 *(__be64 *)&c.ip_lo = *(__be64 *)(lip->s6_addr + 8);

3241 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);

3242 }

3243

3244 /**

3245 * cxgb4_create_server - create an IP server

3246 * @dev: the device

3247 * @stid: the server TID

3248 * @sip: local IP address to bind server to

3249 * @sport: the server's TCP port

3250 * @queue: queue to direct messages from this server to

3251 *

3252 * Create an IP server for the given port and address.

3253 * Returns <0 on error and one of the %NET_XMIT_* values on success.

3254 */

3255 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,

3256 __be32 sip, __be16 sport, __be16 vlan,

3257 unsigned int queue)

3258 {

3259 unsigned int chan;

3260 struct sk_buff *skb;

3261 struct adapter *adap;

3262 struct cpl_pass_open_req *req;

3263 int ret;

3264

3265 skb = alloc_skb(sizeof(*req), GFP_KERNEL);

3266 if (!skb)

3267 return -ENOMEM;

3268

3269 adap = netdev2adap(dev);

3270 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));

3271 INIT_TP_WR(req, 0);

3272 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));

3273 req->local_port = sport;

3274 req->peer_port = htons(0);

3275 req->local_ip = sip;

3276 req->peer_ip = htonl(0);

3277 chan = rxq_to_chan(&adap->sge, queue);

3278 req->opt0 = cpu_to_be64(TX_CHAN(chan));

3279 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |

3280 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));

3281 ret = t4_mgmt_tx(adap, skb);

3282 return net_xmit_eval(ret);

3283 }

3284 EXPORT_SYMBOL(cxgb4_create_server);

3285

3286 /* cxgb4_create_server6 - create an IPv6 server

3287 * @dev: the device

3288 * @stid: the server TID

3289 * @sip: local IPv6 address to bind server to

3290 * @sport: the server's TCP port

3291 * @queue: queue to direct messages from this server to

3292 *

3293 * Create an IPv6 server for the given port and address.

3294 * Returns <0 on error and one of the %NET_XMIT_* values on success.

3295 */

3296 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,

3297 const struct in6_addr *sip, __be16 sport,

3298 unsigned int queue)

3299 {

3300 unsigned int chan;

3301 struct sk_buff *skb;

3302 struct adapter *adap;

3303 struct cpl_pass_open_req6 *req;

3304 int ret;

3305

3306 skb = alloc_skb(sizeof(*req), GFP_KERNEL);

3307 if (!skb)

3308 return -ENOMEM;

3309

3310 adap = netdev2adap(dev);

3311 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));

3312 INIT_TP_WR(req, 0);

3313 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));

3314 req->local_port = sport;

3315 req->peer_port = htons(0);

3316 req->local_ip_hi = *(__be64 *)(sip->s6_addr);

3317 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);

3318 req->peer_ip_hi = cpu_to_be64(0);

3319 req->peer_ip_lo = cpu_to_be64(0);

3320 chan = rxq_to_chan(&adap->sge, queue);

3321 req->opt0 = cpu_to_be64(TX_CHAN(chan));

3322 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |

3323 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));

3324 ret = t4_mgmt_tx(adap, skb);

3325 return net_xmit_eval(ret);

3326 }

3327 EXPORT_SYMBOL(cxgb4_create_server6);

3328

3329 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,

3330 unsigned int queue, bool ipv6)

3331 {

3332 struct sk_buff *skb;

3333 struct adapter *adap;

3334 struct cpl_close_listsvr_req *req;

3335 int ret;

3336

3337 adap = netdev2adap(dev);

3338

3339 skb = alloc_skb(sizeof(*req), GFP_KERNEL);

3340 if (!skb)

3341 return -ENOMEM;

3342

3343 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));

3344 INIT_TP_WR(req, 0);

3345 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));

3346 req->reply_ctrl = htons(NO_REPLY(0) | (ipv6 ? LISTSVR_IPV6(1) :

3347 LISTSVR_IPV6(0)) | QUEUENO(queue));

3348 ret = t4_mgmt_tx(adap, skb);

3349 return net_xmit_eval(ret);

3350 }

3351 EXPORT_SYMBOL(cxgb4_remove_server);

3352

3353 /**

3354 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU

3355 * @mtus: the HW MTU table

3356 * @mtu: the target MTU

3357 * @idx: index of selected entry in the MTU table

3358 *

3359 * Returns the index and the value in the HW MTU table that is closest to

3360 * but does not exceed @mtu, unless @mtu is smaller than any value in the

3361 * table, in which case that smallest available value is selected.

3362 */

3363 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,

3364 unsigned int *idx)

3365 {

3366 unsigned int i = 0;

3367

3368 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)

3369 ++i;

3370 if (idx)

3371 *idx = i;

3372 return mtus[i];

3373 }

3374 EXPORT_SYMBOL(cxgb4_best_mtu);

3375

3376 /**

3377 * cxgb4_port_chan - get the HW channel of a port

3378 * @dev: the net device for the port

3379 *

3380 * Return the HW Tx channel of the given port.

3381 */

3382 unsigned int cxgb4_port_chan(const struct net_device *dev)

3383 {

3384 return netdev2pinfo(dev)->tx_chan;

3385 }

3386 EXPORT_SYMBOL(cxgb4_port_chan);

3387

3388 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)

3389 {

3390 struct adapter *adap = netdev2adap(dev);

3391 u32 v1, v2, lp_count, hp_count;

3392

3393 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);

3394 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);

3395 if (is_t4(adap->params.chip)) {

3396 lp_count = G_LP_COUNT(v1);

3397 hp_count = G_HP_COUNT(v1);

3398 } else {

3399 lp_count = G_LP_COUNT_T5(v1);

3400 hp_count = G_HP_COUNT_T5(v2);

3401 }

3402 return lpfifo ? lp_count : hp_count;

3403 }

3404 EXPORT_SYMBOL(cxgb4_dbfifo_count);

3405

3406 /**

3407 * cxgb4_port_viid - get the VI id of a port

3408 * @dev: the net device for the port

3409 *

3410 * Return the VI id of the given port.

3411 */

3412 unsigned int cxgb4_port_viid(const struct net_device *dev)

3413 {

3414 return netdev2pinfo(dev)->viid;

3415 }

3416 EXPORT_SYMBOL(cxgb4_port_viid);

3417

3418 /**

3419 * cxgb4_port_idx - get the index of a port

3420 * @dev: the net device for the port

3421 *

3422 * Return the index of the given port.

3423 */

3424 unsigned int cxgb4_port_idx(const struct net_device *dev)

3425 {

3426 return netdev2pinfo(dev)->port_id;

3427 }

3428 EXPORT_SYMBOL(cxgb4_port_idx);

3429

3430 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,

3431 struct tp_tcp_stats *v6)

3432 {

3433 struct adapter *adap = pci_get_drvdata(pdev);

3434

3435 spin_lock(&adap->stats_lock);

3436 t4_tp_get_tcp_stats(adap, v4, v6);

3437 spin_unlock(&adap->stats_lock);

3438 }

3439 EXPORT_SYMBOL(cxgb4_get_tcp_stats);

3440

3441 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,

3442 const unsigned int *pgsz_order)

3443 {

3444 struct adapter *adap = netdev2adap(dev);

3445

3446 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);

3447 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |

3448 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |

3449 HPZ3(pgsz_order[3]));

3450 }

3451 EXPORT_SYMBOL(cxgb4_iscsi_init);

3452

3453 int cxgb4_flush_eq_cache(struct net_device *dev)

3454 {

3455 struct adapter *adap = netdev2adap(dev);

3456 int ret;

3457

3458 ret = t4_fwaddrspace_write(adap, adap->mbox,

3459 0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);

3460 return ret;

3461 }

3462 EXPORT_SYMBOL(cxgb4_flush_eq_cache);

3463

3464 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)

3465 {

3466 u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;

3467 __be64 indices;

3468 int ret;

3469

3470 ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);

3471 if (!ret) {

3472 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;

3473 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;

3474 }

3475 return ret;

3476 }

3477

3478 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,

3479 u16 size)

3480 {

3481 struct adapter *adap = netdev2adap(dev);

3482 u16 hw_pidx, hw_cidx;

3483 int ret;

3484

3485 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);

3486 if (ret)

3487 goto out;

3488

3489 if (pidx != hw_pidx) {

3490 u16 delta;

3491

3492 if (pidx >= hw_pidx)

3493 delta = pidx - hw_pidx;

3494 else

3495 delta = size - hw_pidx + pidx;

3496 wmb();

3497 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),

3498 QID(qid) | PIDX(delta));

3499 }

3500 out:

3501 return ret;

3502 }

3503 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);

3504

3505 void cxgb4_disable_db_coalescing(struct net_device *dev)

3506 {

3507 struct adapter *adap;

3508

3509 adap = netdev2adap(dev);

3510 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE,

3511 F_NOCOALESCE);

3512 }

3513 EXPORT_SYMBOL(cxgb4_disable_db_coalescing);

3514

3515 void cxgb4_enable_db_coalescing(struct net_device *dev)

3516 {

3517 struct adapter *adap;

3518

3519 adap = netdev2adap(dev);

3520 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE, 0);

3521 }

3522 EXPORT_SYMBOL(cxgb4_enable_db_coalescing);

3523

3524 static struct pci_driver cxgb4_driver;

3525

3526 static void check_neigh_update(struct neighbour *neigh)

3527 {

3528 const struct device *parent;

3529 const struct net_device *netdev = neigh->dev;

3530

3531 if (netdev->priv_flags & IFF_802_1Q_VLAN)

3532 netdev = vlan_dev_real_dev(netdev);

3533 parent = netdev->dev.parent;

3534 if (parent && parent->driver == &cxgb4_driver.driver)

3535 t4_l2t_update(dev_get_drvdata(parent), neigh);

3536 }

3537

3538 static int netevent_cb(struct notifier_block *nb, unsigned long event,

3539 void *data)

3540 {

3541 switch (event) {

3542 case NETEVENT_NEIGH_UPDATE:

3543 check_neigh_update(data);

3544 break;

3545 case NETEVENT_REDIRECT:

3546 default:

3547 break;

3548 }

3549 return 0;

3550 }

3551

3552 static bool netevent_registered;

3553 static struct notifier_block cxgb4_netevent_nb = {

3554 .notifier_call = netevent_cb

3555 };

3556

3557 static void drain_db_fifo(struct adapter *adap, int usecs)

3558 {

3559 u32 v1, v2, lp_count, hp_count;

3560

3561 do {

3562 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);

3563 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);

3564 if (is_t4(adap->params.chip)) {

3565 lp_count = G_LP_COUNT(v1);

3566 hp_count = G_HP_COUNT(v1);

3567 } else {

3568 lp_count = G_LP_COUNT_T5(v1);

3569 hp_count = G_HP_COUNT_T5(v2);

3570 }

3571

3572 if (lp_count == 0 && hp_count == 0)

3573 break;

3574 set_current_state(TASK_UNINTERRUPTIBLE);

3575 schedule_timeout(usecs_to_jiffies(usecs));

3576 } while (1);

3577 }

3578

3579 static void disable_txq_db(struct sge_txq *q)

3580 {

3581 spin_lock_irq(&q->db_lock);

3582 q->db_disabled = 1;

3583 spin_unlock_irq(&q->db_lock);

3584 }

3585

3586 static void enable_txq_db(struct sge_txq *q)

3587 {

3588 spin_lock_irq(&q->db_lock);

3589 q->db_disabled = 0;

3590 spin_unlock_irq(&q->db_lock);

3591 }

3592

3593 static void disable_dbs(struct adapter *adap)

3594 {

3595 int i;

3596

3597 for_each_ethrxq(&adap->sge, i)

3598 disable_txq_db(&adap->sge.ethtxq[i].q);

3599 for_each_ofldrxq(&adap->sge, i)

3600 disable_txq_db(&adap->sge.ofldtxq[i].q);

3601 for_each_port(adap, i)

3602 disable_txq_db(&adap->sge.ctrlq[i].q);

3603 }

3604

3605 static void enable_dbs(struct adapter *adap)

3606 {

3607 int i;

3608

3609 for_each_ethrxq(&adap->sge, i)

3610 enable_txq_db(&adap->sge.ethtxq[i].q);

3611 for_each_ofldrxq(&adap->sge, i)

3612 enable_txq_db(&adap->sge.ofldtxq[i].q);

3613 for_each_port(adap, i)

3614 enable_txq_db(&adap->sge.ctrlq[i].q);

3615 }

3616

3617 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)

3618 {

3619 u16 hw_pidx, hw_cidx;

3620 int ret;

3621

3622 spin_lock_bh(&q->db_lock);

3623 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);

3624 if (ret)

3625 goto out;

3626 if (q->db_pidx != hw_pidx) {

3627 u16 delta;

3628

3629 if (q->db_pidx >= hw_pidx)

3630 delta = q->db_pidx - hw_pidx;

3631 else

3632 delta = q->size - hw_pidx + q->db_pidx;

3633 wmb();

3634 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),

3635 QID(q->cntxt_id) | PIDX(delta));

3636 }

3637 out:

3638 q->db_disabled = 0;

3639 spin_unlock_bh(&q->db_lock);

3640 if (ret)

3641 CH_WARN(adap, "DB drop recovery failed.\n");

3642 }

3643 static void recover_all_queues(struct adapter *adap)

3644 {

3645 int i;

3646

3647 for_each_ethrxq(&adap->sge, i)

3648 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);

3649 for_each_ofldrxq(&adap->sge, i)

3650 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);

3651 for_each_port(adap, i)

3652 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);

3653 }

3654

3655 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)

3656 {

3657 mutex_lock(&uld_mutex);

3658 if (adap->uld_handle[CXGB4_ULD_RDMA])

3659 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],

3660 cmd);

3661 mutex_unlock(&uld_mutex);

3662 }

3663

3664 static void process_db_full(struct work_struct *work)

3665 {

3666 struct adapter *adap;

3667

3668 adap = container_of(work, struct adapter, db_full_task);

3669

3670 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);

3671 drain_db_fifo(adap, dbfifo_drain_delay);

3672 t4_set_reg_field(adap, SGE_INT_ENABLE3,

3673 DBFIFO_HP_INT | DBFIFO_LP_INT,

3674 DBFIFO_HP_INT | DBFIFO_LP_INT);

3675 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);

3676 }

3677

3678 static void process_db_drop(struct work_struct *work)

3679 {

3680 struct adapter *adap;

3681

3682 adap = container_of(work, struct adapter, db_drop_task);

3683

3684 if (is_t4(adap->params.chip)) {

3685 disable_dbs(adap);

3686 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);

3687 drain_db_fifo(adap, 1);

3688 recover_all_queues(adap);

3689 enable_dbs(adap);

3690 } else {

3691 u32 dropped_db = t4_read_reg(adap, 0x010ac);

3692 u16 qid = (dropped_db >> 15) & 0x1ffff;

3693 u16 pidx_inc = dropped_db & 0x1fff;

3694 unsigned int s_qpp;

3695 unsigned short udb_density;

3696 unsigned long qpshift;

3697 int page;

3698 u32 udb;

3699

3700 dev_warn(adap->pdev_dev,

3701 "Dropped DB 0x%x qid %d bar2 %d coalesce %d pidx %d\n",

3702 dropped_db, qid,

3703 (dropped_db >> 14) & 1,

3704 (dropped_db >> 13) & 1,

3705 pidx_inc);

3706

3707 drain_db_fifo(adap, 1);

3708

3709 s_qpp = QUEUESPERPAGEPF1 * adap->fn;

3710 udb_density = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adap,

3711 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);

3712 qpshift = PAGE_SHIFT - ilog2(udb_density);

3713 udb = qid << qpshift;

3714 udb &= PAGE_MASK;

3715 page = udb / PAGE_SIZE;

3716 udb += (qid - (page * udb_density)) * 128;

3717

3718 writel(PIDX(pidx_inc), adap->bar2 + udb + 8);

3719

3720 /* Re-enable BAR2 WC */

3721 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);

3722 }

3723

3724 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);

3725 }

3726

3727 void t4_db_full(struct adapter *adap)

3728 {

3729 if (is_t4(adap->params.chip)) {

3730 t4_set_reg_field(adap, SGE_INT_ENABLE3,

3731 DBFIFO_HP_INT | DBFIFO_LP_INT, 0);

3732 queue_work(workq, &adap->db_full_task);

3733 }

3734 }

3735

3736 void t4_db_dropped(struct adapter *adap)

3737 {

3738 if (is_t4(adap->params.chip))

3739 queue_work(workq, &adap->db_drop_task);

3740 }

3741

3742 static void uld_attach(struct adapter *adap, unsigned int uld)

3743 {

3744 void *handle;

3745 struct cxgb4_lld_info lli;

3746 unsigned short i;

3747

3748 lli.pdev = adap->pdev;

3749 lli.l2t = adap->l2t;

3750 lli.tids = &adap->tids;

3751 lli.ports = adap->port;

3752 lli.vr = &adap->vres;

3753 lli.mtus = adap->params.mtus;

3754 if (uld == CXGB4_ULD_RDMA) {

3755 lli.rxq_ids = adap->sge.rdma_rxq;

3756 lli.nrxq = adap->sge.rdmaqs;

3757 } else if (uld == CXGB4_ULD_ISCSI) {

3758 lli.rxq_ids = adap->sge.ofld_rxq;

3759 lli.nrxq = adap->sge.ofldqsets;

3760 }

3761 lli.ntxq = adap->sge.ofldqsets;

3762 lli.nchan = adap->params.nports;

3763 lli.nports = adap->params.nports;

3764 lli.wr_cred = adap->params.ofldq_wr_cred;

3765 lli.adapter_type = adap->params.chip;

3766 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));

3767 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(

3768 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>

3769 (adap->fn * 4));

3770 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(

3771 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>

3772 (adap->fn * 4));

3773 lli.filt_mode = adap->params.tp.vlan_pri_map;

3774 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */

3775 for (i = 0; i < NCHAN; i++)

3776 lli.tx_modq[i] = i;

3777 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);

3778 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);

3779 lli.fw_vers = adap->params.fw_vers;

3780 lli.dbfifo_int_thresh = dbfifo_int_thresh;

3781 lli.sge_pktshift = adap->sge.pktshift;

3782 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;

3783 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;

3784

3785 handle = ulds[uld].add(&lli);

3786 if (IS_ERR(handle)) {

3787 dev_warn(adap->pdev_dev,

3788 "could not attach to the %s driver, error %ld\n",

3789 uld_str[uld], PTR_ERR(handle));

3790 return;

3791 }

3792

3793 adap->uld_handle[uld] = handle;

3794

3795 if (!netevent_registered) {

3796 register_netevent_notifier(&cxgb4_netevent_nb);

3797 netevent_registered = true;

3798 }

3799

3800 if (adap->flags & FULL_INIT_DONE)

3801 ulds[uld].state_change(handle, CXGB4_STATE_UP);

3802 }

3803

3804 static void attach_ulds(struct adapter *adap)

3805 {

3806 unsigned int i;

3807

3808 spin_lock(&adap_rcu_lock);

3809 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);

3810 spin_unlock(&adap_rcu_lock);

3811

3812 mutex_lock(&uld_mutex);

3813 list_add_tail(&adap->list_node, &adapter_list);

3814 for (i = 0; i < CXGB4_ULD_MAX; i++)

3815 if (ulds[i].add)

3816 uld_attach(adap, i);

3817 mutex_unlock(&uld_mutex);

3818 }

3819

3820 static void detach_ulds(struct adapter *adap)

3821 {

3822 unsigned int i;

3823

3824 mutex_lock(&uld_mutex);

3825 list_del(&adap->list_node);

3826 for (i = 0; i < CXGB4_ULD_MAX; i++)

3827 if (adap->uld_handle[i]) {

3828 ulds[i].state_change(adap->uld_handle[i],

3829 CXGB4_STATE_DETACH);

3830 adap->uld_handle[i] = NULL;

3831 }

3832 if (netevent_registered && list_empty(&adapter_list)) {

3833 unregister_netevent_notifier(&cxgb4_netevent_nb);

3834 netevent_registered = false;

3835 }

3836 mutex_unlock(&uld_mutex);

3837

3838 spin_lock(&adap_rcu_lock);

3839 list_del_rcu(&adap->rcu_node);

3840 spin_unlock(&adap_rcu_lock);

3841 }

3842

3843 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)

3844 {

3845 unsigned int i;

3846

3847 mutex_lock(&uld_mutex);

3848 for (i = 0; i < CXGB4_ULD_MAX; i++)

3849 if (adap->uld_handle[i])

3850 ulds[i].state_change(adap->uld_handle[i], new_state);

3851 mutex_unlock(&uld_mutex);

3852 }

3853

3854 /**

3855 * cxgb4_register_uld - register an upper-layer driver

3856 * @type: the ULD type

3857 * @p: the ULD methods

3858 *

3859 * Registers an upper-layer driver with this driver and notifies the ULD

3860 * about any presently available devices that support its type. Returns

3861 * %-EBUSY if a ULD of the same type is already registered.

3862 */

3863 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)

3864 {

3865 int ret = 0;

3866 struct adapter *adap;

3867

3868 if (type >= CXGB4_ULD_MAX)

3869 return -EINVAL;

3870 mutex_lock(&uld_mutex);

3871 if (ulds[type].add) {

3872 ret = -EBUSY;

3873 goto out;

3874 }

3875 ulds[type] = *p;

3876 list_for_each_entry(adap, &adapter_list, list_node)

3877 uld_attach(adap, type);

3878 out: mutex_unlock(&uld_mutex);

3879 return ret;

3880 }

3881 EXPORT_SYMBOL(cxgb4_register_uld);

3882

3883 /**

3884 * cxgb4_unregister_uld - unregister an upper-layer driver

3885 * @type: the ULD type

3886 *

3887 * Unregisters an existing upper-layer driver.

3888 */

3889 int cxgb4_unregister_uld(enum cxgb4_uld type)

3890 {

3891 struct adapter *adap;

3892

3893 if (type >= CXGB4_ULD_MAX)

3894 return -EINVAL;

3895 mutex_lock(&uld_mutex);

3896 list_for_each_entry(adap, &adapter_list, list_node)

3897 adap->uld_handle[type] = NULL;

3898 ulds[type].add = NULL;

3899 mutex_unlock(&uld_mutex);

3900 return 0;

3901 }

3902 EXPORT_SYMBOL(cxgb4_unregister_uld);

3903

3904 /* Check if netdev on which event is occured belongs to us or not. Return

3905 * suceess (1) if it belongs otherwise failure (0).

3906 */

3907 static int cxgb4_netdev(struct net_device *netdev)

3908 {

3909 struct adapter *adap;

3910 int i;

3911

3912 spin_lock(&adap_rcu_lock);

3913 list_for_each_entry_rcu(adap, &adap_rcu_list, rcu_node)

3914 for (i = 0; i < MAX_NPORTS; i++)

3915 if (adap->port[i] == netdev) {

3916 spin_unlock(&adap_rcu_lock);

3917 return 1;

3918 }

3919 spin_unlock(&adap_rcu_lock);

3920 return 0;

3921 }

3922

3923 static int clip_add(struct net_device *event_dev, struct inet6_ifaddr *ifa,

3924 unsigned long event)

3925 {

3926 int ret = NOTIFY_DONE;

3927

3928 rcu_read_lock();

3929 if (cxgb4_netdev(event_dev)) {

3930 switch (event) {

3931 case NETDEV_UP:

3932 ret = cxgb4_clip_get(event_dev,

3933 (const struct in6_addr *)ifa->addr.s6_addr);

3934 if (ret < 0) {

3935 rcu_read_unlock();

3936 return ret;

3937 }

3938 ret = NOTIFY_OK;

3939 break;

3940 case NETDEV_DOWN:

3941 cxgb4_clip_release(event_dev,

3942 (const struct in6_addr *)ifa->addr.s6_addr);

3943 ret = NOTIFY_OK;

3944 break;

3945 default:

3946 break;

3947 }

3948 }

3949 rcu_read_unlock();

3950 return ret;

3951 }

3952

3953 static int cxgb4_inet6addr_handler(struct notifier_block *this,

3954 unsigned long event, void *data)

3955 {

3956 struct inet6_ifaddr *ifa = data;

3957 struct net_device *event_dev;

3958 int ret = NOTIFY_DONE;

3959 struct bonding *bond = netdev_priv(ifa->idev->dev);

3960 struct list_head *iter;

3961 struct slave *slave;

3962 struct pci_dev *first_pdev = NULL;

3963

3964 if (ifa->idev->dev->priv_flags & IFF_802_1Q_VLAN) {

3965 event_dev = vlan_dev_real_dev(ifa->idev->dev);

3966 ret = clip_add(event_dev, ifa, event);

3967 } else if (ifa->idev->dev->flags & IFF_MASTER) {

3968 /* It is possible that two different adapters are bonded in one

3969 * bond. We need to find such different adapters and add clip

3970 * in all of them only once.

3971 */

3972 read_lock(&bond->lock);

3973 bond_for_each_slave(bond, slave, iter) {

3974 if (!first_pdev) {

3975 ret = clip_add(slave->dev, ifa, event);

3976 /* If clip_add is success then only initialize

3977 * first_pdev since it means it is our device

3978 */

3979 if (ret == NOTIFY_OK)

3980 first_pdev = to_pci_dev(

3981 slave->dev->dev.parent);

3982 } else if (first_pdev !=

3983 to_pci_dev(slave->dev->dev.parent))

3984 ret = clip_add(slave->dev, ifa, event);

3985 }

3986 read_unlock(&bond->lock);

3987 } else

3988 ret = clip_add(ifa->idev->dev, ifa, event);

3989

3990 return ret;

3991 }

3992

3993 static struct notifier_block cxgb4_inet6addr_notifier = {

3994 .notifier_call = cxgb4_inet6addr_handler

3995 };

3996

3997 /* Retrieves IPv6 addresses from a root device (bond, vlan) associated with

3998 * a physical device.

3999 * The physical device reference is needed to send the actul CLIP command.

4000 */

4001 static int update_dev_clip(struct net_device *root_dev, struct net_device *dev)

4002 {

4003 struct inet6_dev *idev = NULL;

4004 struct inet6_ifaddr *ifa;

4005 int ret = 0;

4006

4007 idev = __in6_dev_get(root_dev);

4008 if (!idev)

4009 return ret;

4010

4011 read_lock_bh(&idev->lock);

4012 list_for_each_entry(ifa, &idev->addr_list, if_list) {

4013 ret = cxgb4_clip_get(dev,

4014 (const struct in6_addr *)ifa->addr.s6_addr);

4015 if (ret < 0)

4016 break;

4017 }

4018 read_unlock_bh(&idev->lock);

4019

4020 return ret;

4021 }

4022

4023 static int update_root_dev_clip(struct net_device *dev)

4024 {

4025 struct net_device *root_dev = NULL;

4026 int i, ret = 0;

4027

4028 /* First populate the real net device's IPv6 addresses */

4029 ret = update_dev_clip(dev, dev);

4030 if (ret)

4031 return ret;

4032

4033 /* Parse all bond and vlan devices layered on top of the physical dev */

4034 for (i = 0; i < VLAN_N_VID; i++) {

4035 root_dev = __vlan_find_dev_deep(dev, htons(ETH_P_8021Q), i);

4036 if (!root_dev)

4037 continue;

4038

4039 ret = update_dev_clip(root_dev, dev);

4040 if (ret)

4041 break;

4042 }

4043 return ret;

4044 }

4045

4046 static void update_clip(const struct adapter *adap)

4047 {

4048 int i;

4049 struct net_device *dev;

4050 int ret;

4051

4052 rcu_read_lock();

4053

4054 for (i = 0; i < MAX_NPORTS; i++) {

4055 dev = adap->port[i];

4056 ret = 0;

4057

4058 if (dev)

4059 ret = update_root_dev_clip(dev);

4060

4061 if (ret < 0)

4062 break;

4063 }

4064 rcu_read_unlock();

4065 }

4066

4067 /**

4068 * cxgb_up - enable the adapter

4069 * @adap: adapter being enabled

4070 *

4071 * Called when the first port is enabled, this function performs the

4072 * actions necessary to make an adapter operational, such as completing

4073 * the initialization of HW modules, and enabling interrupts.

4074 *

4075 * Must be called with the rtnl lock held.

4076 */

4077 static int cxgb_up(struct adapter *adap)

4078 {

4079 int err;

4080

4081 err = setup_sge_queues(adap);

4082 if (err)

4083 goto out;

4084 err = setup_rss(adap);

4085 if (err)

4086 goto freeq;

4087

4088 if (adap->flags & USING_MSIX) {

4089 name_msix_vecs(adap);

4090 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,

4091 adap->msix_info[0].desc, adap);

4092 if (err)

4093 goto irq_err;

4094

4095 err = request_msix_queue_irqs(adap);

4096 if (err) {

4097 free_irq(adap->msix_info[0].vec, adap);

4098 goto irq_err;

4099 }

4100 } else {

4101 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),

4102 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,

4103 adap->port[0]->name, adap);

4104 if (err)

4105 goto irq_err;

4106 }

4107 enable_rx(adap);

4108 t4_sge_start(adap);

4109 t4_intr_enable(adap);

4110 adap->flags |= FULL_INIT_DONE;

4111 notify_ulds(adap, CXGB4_STATE_UP);

4112 update_clip(adap);

4113 out:

4114 return err;

4115 irq_err:

4116 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);

4117 freeq:

4118 t4_free_sge_resources(adap);

4119 goto out;

4120 }

4121

4122 static void cxgb_down(struct adapter *adapter)

4123 {

4124 t4_intr_disable(adapter);

4125 cancel_work_sync(&adapter->tid_release_task);

4126 cancel_work_sync(&adapter->db_full_task);

4127 cancel_work_sync(&adapter->db_drop_task);

4128 adapter->tid_release_task_busy = false;

4129 adapter->tid_release_head = NULL;

4130

4131 if (adapter->flags & USING_MSIX) {

4132 free_msix_queue_irqs(adapter);

4133 free_irq(adapter->msix_info[0].vec, adapter);

4134 } else

4135 free_irq(adapter->pdev->irq, adapter);

4136 quiesce_rx(adapter);

4137 t4_sge_stop(adapter);

4138 t4_free_sge_resources(adapter);

4139 adapter->flags &= ~FULL_INIT_DONE;

4140 }

4141

4142 /*

4143 * net_device operations

4144 */

4145 static int cxgb_open(struct net_device *dev)

4146 {

4147 int err;

4148 struct port_info *pi = netdev_priv(dev);

4149 struct adapter *adapter = pi->adapter;

4150

4151 netif_carrier_off(dev);

4152

4153 if (!(adapter->flags & FULL_INIT_DONE)) {

4154 err = cxgb_up(adapter);

4155 if (err < 0)

4156 return err;

4157 }

4158

4159 err = link_start(dev);

4160 if (!err)

4161 netif_tx_start_all_queues(dev);

4162 return err;

4163 }

4164

4165 static int cxgb_close(struct net_device *dev)

4166 {

4167 struct port_info *pi = netdev_priv(dev);

4168 struct adapter *adapter = pi->adapter;

4169

4170 netif_tx_stop_all_queues(dev);

4171 netif_carrier_off(dev);

4172 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);

4173 }

4174

4175 /* Return an error number if the indicated filter isn't writable ...

4176 */

4177 static int writable_filter(struct filter_entry *f)

4178 {

4179 if (f->locked)

4180 return -EPERM;

4181 if (f->pending)

4182 return -EBUSY;

4183

4184 return 0;

4185 }

4186

4187 /* Delete the filter at the specified index (if valid). The checks for all

4188 * the common problems with doing this like the filter being locked, currently

4189 * pending in another operation, etc.

4190 */

4191 static int delete_filter(struct adapter *adapter, unsigned int fidx)

4192 {

4193 struct filter_entry *f;

4194 int ret;

4195

4196 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)

4197 return -EINVAL;

4198

4199 f = &adapter->tids.ftid_tab[fidx];

4200 ret = writable_filter(f);

4201 if (ret)

4202 return ret;

4203 if (f->valid)

4204 return del_filter_wr(adapter, fidx);

4205

4206 return 0;

4207 }

4208

4209 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,

4210 __be32 sip, __be16 sport, __be16 vlan,

4211 unsigned int queue, unsigned char port, unsigned char mask)

4212 {

4213 int ret;

4214 struct filter_entry *f;

4215 struct adapter *adap;

4216 int i;

4217 u8 *val;

4218

4219 adap = netdev2adap(dev);

4220

4221 /* Adjust stid to correct filter index */

4222 stid -= adap->tids.sftid_base;

4223 stid += adap->tids.nftids;

4224

4225 /* Check to make sure the filter requested is writable ...

4226 */

4227 f = &adap->tids.ftid_tab[stid];

4228 ret = writable_filter(f);

4229 if (ret)

4230 return ret;

4231

4232 /* Clear out any old resources being used by the filter before

4233 * we start constructing the new filter.

4234 */

4235 if (f->valid)

4236 clear_filter(adap, f);

4237

4238 /* Clear out filter specifications */

4239 memset(&f->fs, 0, sizeof(struct ch_filter_specification));

4240 f->fs.val.lport = cpu_to_be16(sport);

4241 f->fs.mask.lport = ~0;

4242 val = (u8 *)&sip;

4243 if ((val[0] | val[1] | val[2] | val[3]) != 0) {

4244 for (i = 0; i < 4; i++) {

4245 f->fs.val.lip[i] = val[i];

4246 f->fs.mask.lip[i] = ~0;

4247 }

4248 if (adap->params.tp.vlan_pri_map & F_PORT) {

4249 f->fs.val.iport = port;

4250 f->fs.mask.iport = mask;

4251 }

4252 }

4253

4254 if (adap->params.tp.vlan_pri_map & F_PROTOCOL) {

4255 f->fs.val.proto = IPPROTO_TCP;

4256 f->fs.mask.proto = ~0;

4257 }

4258

4259 f->fs.dirsteer = 1;

4260 f->fs.iq = queue;

4261 /* Mark filter as locked */

4262 f->locked = 1;

4263 f->fs.rpttid = 1;

4264

4265 ret = set_filter_wr(adap, stid);

4266 if (ret) {

4267 clear_filter(adap, f);

4268 return ret;

4269 }

4270

4271 return 0;

4272 }

4273 EXPORT_SYMBOL(cxgb4_create_server_filter);

4274

4275 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,

4276 unsigned int queue, bool ipv6)

4277 {

4278 int ret;

4279 struct filter_entry *f;

4280 struct adapter *adap;

4281

4282 adap = netdev2adap(dev);

4283

4284 /* Adjust stid to correct filter index */

4285 stid -= adap->tids.sftid_base;

4286 stid += adap->tids.nftids;

4287

4288 f = &adap->tids.ftid_tab[stid];

4289 /* Unlock the filter */

4290 f->locked = 0;

4291

4292 ret = delete_filter(adap, stid);

4293 if (ret)

4294 return ret;

4295

4296 return 0;

4297 }

4298 EXPORT_SYMBOL(cxgb4_remove_server_filter);

4299

4300 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,

4301 struct rtnl_link_stats64 *ns)

4302 {

4303 struct port_stats stats;

4304 struct port_info *p = netdev_priv(dev);

4305 struct adapter *adapter = p->adapter;

4306

4307 /* Block retrieving statistics during EEH error

4308 * recovery. Otherwise, the recovery might fail

4309 * and the PCI device will be removed permanently

4310 */

4311 spin_lock(&adapter->stats_lock);

4312 if (!netif_device_present(dev)) {

4313 spin_unlock(&adapter->stats_lock);

4314 return ns;

4315 }

4316 t4_get_port_stats(adapter, p->tx_chan, &stats);

4317 spin_unlock(&adapter->stats_lock);

4318

4319 ns->tx_bytes = stats.tx_octets;

4320 ns->tx_packets = stats.tx_frames;

4321 ns->rx_bytes = stats.rx_octets;

4322 ns->rx_packets = stats.rx_frames;

4323 ns->multicast = stats.rx_mcast_frames;

4324

4325 /* detailed rx_errors */

4326 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +

4327 stats.rx_runt;

4328 ns->rx_over_errors = 0;

4329 ns->rx_crc_errors = stats.rx_fcs_err;

4330 ns->rx_frame_errors = stats.rx_symbol_err;

4331 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +

4332 stats.rx_ovflow2 + stats.rx_ovflow3 +

4333 stats.rx_trunc0 + stats.rx_trunc1 +

4334 stats.rx_trunc2 + stats.rx_trunc3;

4335 ns->rx_missed_errors = 0;

4336

4337 /* detailed tx_errors */

4338 ns->tx_aborted_errors = 0;

4339 ns->tx_carrier_errors = 0;

4340 ns->tx_fifo_errors = 0;

4341 ns->tx_heartbeat_errors = 0;

4342 ns->tx_window_errors = 0;

4343

4344 ns->tx_errors = stats.tx_error_frames;

4345 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +

4346 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;

4347 return ns;

4348 }

4349

4350 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)

4351 {

4352 unsigned int mbox;

4353 int ret = 0, prtad, devad;

4354 struct port_info *pi = netdev_priv(dev);

4355 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;

4356

4357 switch (cmd) {

4358 case SIOCGMIIPHY:

4359 if (pi->mdio_addr < 0)

4360 return -EOPNOTSUPP;

4361 data->phy_id = pi->mdio_addr;

4362 break;

4363 case SIOCGMIIREG:

4364 case SIOCSMIIREG:

4365 if (mdio_phy_id_is_c45(data->phy_id)) {

4366 prtad = mdio_phy_id_prtad(data->phy_id);

4367 devad = mdio_phy_id_devad(data->phy_id);

4368 } else if (data->phy_id < 32) {

4369 prtad = data->phy_id;

4370 devad = 0;

4371 data->reg_num &= 0x1f;

4372 } else

4373 return -EINVAL;

4374

4375 mbox = pi->adapter->fn;

4376 if (cmd == SIOCGMIIREG)

4377 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,

4378 data->reg_num, &data->val_out);

4379 else

4380 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,

4381 data->reg_num, data->val_in);

4382 break;

4383 default:

4384 return -EOPNOTSUPP;

4385 }

4386 return ret;

4387 }

4388

4389 static void cxgb_set_rxmode(struct net_device *dev)

4390 {

4391 /* unfortunately we can't return errors to the stack */

4392 set_rxmode(dev, -1, false);

4393 }

4394

4395 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)

4396 {

4397 int ret;

4398 struct port_info *pi = netdev_priv(dev);

4399

4400 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */

4401 return -EINVAL;

4402 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,

4403 -1, -1, -1, true);

4404 if (!ret)

4405 dev->mtu = new_mtu;

4406 return ret;

4407 }

4408

4409 static int cxgb_set_mac_addr(struct net_device *dev, void *p)

4410 {

4411 int ret;

4412 struct sockaddr *addr = p;

4413 struct port_info *pi = netdev_priv(dev);

4414

4415 if (!is_valid_ether_addr(addr->sa_data))

4416 return -EADDRNOTAVAIL;

4417

4418 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,

4419 pi->xact_addr_filt, addr->sa_data, true, true);

4420 if (ret < 0)

4421 return ret;

4422

4423 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);

4424 pi->xact_addr_filt = ret;

4425 return 0;

4426 }

4427

4428 #ifdef CONFIG_NET_POLL_CONTROLLER

4429 static void cxgb_netpoll(struct net_device *dev)

4430 {

4431 struct port_info *pi = netdev_priv(dev);

4432 struct adapter *adap = pi->adapter;

4433

4434 if (adap->flags & USING_MSIX) {

4435 int i;

4436 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];

4437

4438 for (i = pi->nqsets; i; i--, rx++)

4439 t4_sge_intr_msix(0, &rx->rspq);

4440 } else

4441 t4_intr_handler(adap)(0, adap);

4442 }

4443 #endif

4444

4445 static const struct net_device_ops cxgb4_netdev_ops = {

4446 .ndo_open = cxgb_open,

4447 .ndo_stop = cxgb_close,

4448 .ndo_start_xmit = t4_eth_xmit,

4449 .ndo_get_stats64 = cxgb_get_stats,

4450 .ndo_set_rx_mode = cxgb_set_rxmode,

4451 .ndo_set_mac_address = cxgb_set_mac_addr,

4452 .ndo_set_features = cxgb_set_features,

4453 .ndo_validate_addr = eth_validate_addr,

4454 .ndo_do_ioctl = cxgb_ioctl,

4455 .ndo_change_mtu = cxgb_change_mtu,

4456 #ifdef CONFIG_NET_POLL_CONTROLLER

4457 .ndo_poll_controller = cxgb_netpoll,

4458 #endif

4459 };

4460

4461 void t4_fatal_err(struct adapter *adap)

4462 {

4463 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);

4464 t4_intr_disable(adap);

4465 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");

4466 }

4467

4468 static void setup_memwin(struct adapter *adap)

4469 {

4470 u32 bar0, mem_win0_base, mem_win1_base, mem_win2_base;

4471

4472 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */

4473 if (is_t4(adap->params.chip)) {

4474 mem_win0_base = bar0 + MEMWIN0_BASE;

4475 mem_win1_base = bar0 + MEMWIN1_BASE;

4476 mem_win2_base = bar0 + MEMWIN2_BASE;

4477 } else {

4478 /* For T5, only relative offset inside the PCIe BAR is passed */

4479 mem_win0_base = MEMWIN0_BASE;

4480 mem_win1_base = MEMWIN1_BASE_T5;

4481 mem_win2_base = MEMWIN2_BASE_T5;

4482 }

4483 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),

4484 mem_win0_base | BIR(0) |

4485 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));

4486 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),

4487 mem_win1_base | BIR(0) |

4488 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));

4489 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),

4490 mem_win2_base | BIR(0) |

4491 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));

4492 }

4493

4494 static void setup_memwin_rdma(struct adapter *adap)

4495 {

4496 if (adap->vres.ocq.size) {

4497 unsigned int start, sz_kb;

4498

4499 start = pci_resource_start(adap->pdev, 2) +

4500 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);

4501 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;

4502 t4_write_reg(adap,

4503 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),

4504 start | BIR(1) | WINDOW(ilog2(sz_kb)));

4505 t4_write_reg(adap,

4506 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),

4507 adap->vres.ocq.start);

4508 t4_read_reg(adap,

4509 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));

4510 }

4511 }

4512

4513 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)

4514 {

4515 u32 v;

4516 int ret;

4517

4518 /* get device capabilities */

4519 memset(c, 0, sizeof(*c));

4520 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |

4521 FW_CMD_REQUEST | FW_CMD_READ);

4522 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));

4523 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);

4524 if (ret < 0)

4525 return ret;

4526

4527 /* select capabilities we'll be using */

4528 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {

4529 if (!vf_acls)

4530 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);

4531 else

4532 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);

4533 } else if (vf_acls) {

4534 dev_err(adap->pdev_dev, "virtualization ACLs not supported");

4535 return ret;

4536 }

4537 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |

4538 FW_CMD_REQUEST | FW_CMD_WRITE);

4539 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);

4540 if (ret < 0)

4541 return ret;

4542

4543 ret = t4_config_glbl_rss(adap, adap->fn,

4544 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,

4545 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |

4546 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);

4547 if (ret < 0)

4548 return ret;

4549

4550 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,

4551 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);

4552 if (ret < 0)

4553 return ret;

4554

4555 t4_sge_init(adap);

4556

4557 /* tweak some settings */

4558 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);

4559 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));

4560 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);

4561 v = t4_read_reg(adap, TP_PIO_DATA);

4562 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);

4563

4564 /* first 4 Tx modulation queues point to consecutive Tx channels */

4565 adap->params.tp.tx_modq_map = 0xE4;

4566 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP,

4567 V_TX_MOD_QUEUE_REQ_MAP(adap->params.tp.tx_modq_map));

4568

4569 /* associate each Tx modulation queue with consecutive Tx channels */

4570 v = 0x84218421;

4571 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,

4572 &v, 1, A_TP_TX_SCHED_HDR);

4573 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,

4574 &v, 1, A_TP_TX_SCHED_FIFO);

4575 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,

4576 &v, 1, A_TP_TX_SCHED_PCMD);

4577

4578 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */

4579 if (is_offload(adap)) {

4580 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0,

4581 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |

4582 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |

4583 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |

4584 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));

4585 t4_write_reg(adap, A_TP_TX_MOD_CHANNEL_WEIGHT,

4586 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |

4587 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |

4588 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |

4589 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));

4590 }

4591

4592 /* get basic stuff going */

4593 return t4_early_init(adap, adap->fn);

4594 }

4595

4596 /*

4597 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.

4598 */

4599 #define MAX_ATIDS 8192U

4600

4601 /*

4602 * Phase 0 of initialization: contact FW, obtain config, perform basic init.

4603 *

4604 * If the firmware we're dealing with has Configuration File support, then

4605 * we use that to perform all configuration

4606 */

4607

4608 /*

4609 * Tweak configuration based on module parameters, etc. Most of these have

4610 * defaults assigned to them by Firmware Configuration Files (if we're using

4611 * them) but need to be explicitly set if we're using hard-coded

4612 * initialization. But even in the case of using Firmware Configuration

4613 * Files, we'd like to expose the ability to change these via module

4614 * parameters so these are essentially common tweaks/settings for

4615 * Configuration Files and hard-coded initialization ...

4616 */

4617 static int adap_init0_tweaks(struct adapter *adapter)

4618 {

4619 /*

4620 * Fix up various Host-Dependent Parameters like Page Size, Cache

4621 * Line Size, etc. The firmware default is for a 4KB Page Size and

4622 * 64B Cache Line Size ...

4623 */

4624 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);

4625

4626 /*

4627 * Process module parameters which affect early initialization.

4628 */

4629 if (rx_dma_offset != 2 && rx_dma_offset != 0) {

4630 dev_err(&adapter->pdev->dev,

4631 "Ignoring illegal rx_dma_offset=%d, using 2\n",

4632 rx_dma_offset);

4633 rx_dma_offset = 2;

4634 }

4635 t4_set_reg_field(adapter, SGE_CONTROL,

4636 PKTSHIFT_MASK,

4637 PKTSHIFT(rx_dma_offset));

4638

4639 /*

4640 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux

4641 * adds the pseudo header itself.

4642 */

4643 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,

4644 CSUM_HAS_PSEUDO_HDR, 0);

4645

4646 return 0;

4647 }

4648

4649 /*

4650 * Attempt to initialize the adapter via a Firmware Configuration File.

4651 */

4652 static int adap_init0_config(struct adapter *adapter, int reset)

4653 {

4654 struct fw_caps_config_cmd caps_cmd;

4655 const struct firmware *cf;

4656 unsigned long mtype = 0, maddr = 0;

4657 u32 finiver, finicsum, cfcsum;

4658 int ret;

4659 int config_issued = 0;

4660 char *fw_config_file, fw_config_file_path[256];

4661 char *config_name = NULL;

4662

4663 /*

4664 * Reset device if necessary.

4665 */

4666 if (reset) {

4667 ret = t4_fw_reset(adapter, adapter->mbox,

4668 PIORSTMODE | PIORST);

4669 if (ret < 0)

4670 goto bye;

4671 }

4672

4673 /*

4674 * If we have a T4 configuration file under /lib/firmware/cxgb4/,

4675 * then use that. Otherwise, use the configuration file stored

4676 * in the adapter flash ...

4677 */

4678 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {

4679 case CHELSIO_T4:

4680 fw_config_file = FW4_CFNAME;

4681 break;

4682 case CHELSIO_T5:

4683 fw_config_file = FW5_CFNAME;

4684 break;

4685 default:

4686 dev_err(adapter->pdev_dev, "Device %d is not supported\n",

4687 adapter->pdev->device);

4688 ret = -EINVAL;

4689 goto bye;

4690 }

4691

4692 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);

4693 if (ret < 0) {

4694 config_name = "On FLASH";

4695 mtype = FW_MEMTYPE_CF_FLASH;

4696 maddr = t4_flash_cfg_addr(adapter);

4697 } else {

4698 u32 params[7], val[7];

4699

4700 sprintf(fw_config_file_path,

4701 "/lib/firmware/%s", fw_config_file);

4702 config_name = fw_config_file_path;

4703

4704 if (cf->size >= FLASH_CFG_MAX_SIZE)

4705 ret = -ENOMEM;

4706 else {

4707 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |

4708 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));

4709 ret = t4_query_params(adapter, adapter->mbox,

4710 adapter->fn, 0, 1, params, val);

4711 if (ret == 0) {

4712 /*

4713 * For t4_memory_write() below addresses and

4714 * sizes have to be in terms of multiples of 4

4715 * bytes. So, if the Configuration File isn't

4716 * a multiple of 4 bytes in length we'll have

4717 * to write that out separately since we can't

4718 * guarantee that the bytes following the

4719 * residual byte in the buffer returned by

4720 * request_firmware() are zeroed out ...

4721 */

4722 size_t resid = cf->size & 0x3;

4723 size_t size = cf->size & ~0x3;

4724 __be32 *data = (__be32 *)cf->data;

4725

4726 mtype = FW_PARAMS_PARAM_Y_GET(val[0]);

4727 maddr = FW_PARAMS_PARAM_Z_GET(val[0]) << 16;

4728

4729 ret = t4_memory_write(adapter, mtype, maddr,

4730 size, data);

4731 if (ret == 0 && resid != 0) {

4732 union {

4733 __be32 word;

4734 char buf[4];

4735 } last;

4736 int i;

4737

4738 last.word = data[size >> 2];

4739 for (i = resid; i < 4; i++)

4740 last.buf[i] = 0;

4741 ret = t4_memory_write(adapter, mtype,

4742 maddr + size,

4743 4, &last.word);

4744 }

4745 }

4746 }

4747

4748 release_firmware(cf);

4749 if (ret)

4750 goto bye;

4751 }

4752

4753 /*

4754 * Issue a Capability Configuration command to the firmware to get it

4755 * to parse the Configuration File. We don't use t4_fw_config_file()

4756 * because we want the ability to modify various features after we've

4757 * processed the configuration file ...

4758 */

4759 memset(&caps_cmd, 0, sizeof(caps_cmd));

4760 caps_cmd.op_to_write =

4761 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |

4762 FW_CMD_REQUEST |

4763 FW_CMD_READ);

4764 caps_cmd.cfvalid_to_len16 =

4765 htonl(FW_CAPS_CONFIG_CMD_CFVALID |

4766 FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |

4767 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |

4768 FW_LEN16(caps_cmd));

4769 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),

4770 &caps_cmd);

4771

4772 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware

4773 * Configuration File in FLASH), our last gasp effort is to use the

4774 * Firmware Configuration File which is embedded in the firmware. A

4775 * very few early versions of the firmware didn't have one embedded

4776 * but we can ignore those.

4777 */

4778 if (ret == -ENOENT) {

4779 memset(&caps_cmd, 0, sizeof(caps_cmd));

4780 caps_cmd.op_to_write =

4781 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |

4782 FW_CMD_REQUEST |

4783 FW_CMD_READ);

4784 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));

4785 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,

4786 sizeof(caps_cmd), &caps_cmd);

4787 config_name = "Firmware Default";

4788 }

4789

4790 config_issued = 1;

4791 if (ret < 0)

4792 goto bye;

4793

4794 finiver = ntohl(caps_cmd.finiver);

4795 finicsum = ntohl(caps_cmd.finicsum);

4796 cfcsum = ntohl(caps_cmd.cfcsum);

4797 if (finicsum != cfcsum)

4798 dev_warn(adapter->pdev_dev, "Configuration File checksum "\

4799 "mismatch: [fini] csum=%#x, computed csum=%#x\n",

4800 finicsum, cfcsum);

4801

4802 /*

4803 * And now tell the firmware to use the configuration we just loaded.

4804 */

4805 caps_cmd.op_to_write =

4806 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |

4807 FW_CMD_REQUEST |

4808 FW_CMD_WRITE);

4809 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));

4810 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),

4811 NULL);

4812 if (ret < 0)

4813 goto bye;

4814

4815 /*

4816 * Tweak configuration based on system architecture, module

4817 * parameters, etc.

4818 */

4819 ret = adap_init0_tweaks(adapter);

4820 if (ret < 0)

4821 goto bye;

4822

4823 /*

4824 * And finally tell the firmware to initialize itself using the

4825 * parameters from the Configuration File.

4826 */

4827 ret = t4_fw_initialize(adapter, adapter->mbox);

4828 if (ret < 0)

4829 goto bye;

4830

4831 /*

4832 * Return successfully and note that we're operating with parameters

4833 * not supplied by the driver, rather than from hard-wired

4834 * initialization constants burried in the driver.

4835 */

4836 adapter->flags |= USING_SOFT_PARAMS;

4837 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\

4838 "Configuration File \"%s\", version %#x, computed checksum %#x\n",

4839 config_name, finiver, cfcsum);

4840 return 0;

4841

4842 /*

4843 * Something bad happened. Return the error ... (If the "error"

4844 * is that there's no Configuration File on the adapter we don't

4845 * want to issue a warning since this is fairly common.)

4846 */

4847 bye:

4848 if (config_issued && ret != -ENOENT)

4849 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",

4850 config_name, -ret);

4851 return ret;

4852 }

4853

4854 /*

4855 * Attempt to initialize the adapter via hard-coded, driver supplied

4856 * parameters ...

4857 */

4858 static int adap_init0_no_config(struct adapter *adapter, int reset)

4859 {

4860 struct sge *s = &adapter->sge;

4861 struct fw_caps_config_cmd caps_cmd;

4862 u32 v;

4863 int i, ret;

4864

4865 /*

4866 * Reset device if necessary

4867 */

4868 if (reset) {

4869 ret = t4_fw_reset(adapter, adapter->mbox,

4870 PIORSTMODE | PIORST);

4871 if (ret < 0)

4872 goto bye;

4873 }

4874

4875 /*

4876 * Get device capabilities and select which we'll be using.

4877 */

4878 memset(&caps_cmd, 0, sizeof(caps_cmd));

4879 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |

4880 FW_CMD_REQUEST | FW_CMD_READ);

4881 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));

4882 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),

4883 &caps_cmd);

4884 if (ret < 0)

4885 goto bye;

4886

4887 if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {

4888 if (!vf_acls)

4889 caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);

4890 else

4891 caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);

4892 } else if (vf_acls) {

4893 dev_err(adapter->pdev_dev, "virtualization ACLs not supported");

4894 goto bye;

4895 }

4896 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |

4897 FW_CMD_REQUEST | FW_CMD_WRITE);

4898 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),

4899 NULL);

4900 if (ret < 0)

4901 goto bye;

4902

4903 /*

4904 * Tweak configuration based on system architecture, module

4905 * parameters, etc.

4906 */

4907 ret = adap_init0_tweaks(adapter);

4908 if (ret < 0)

4909 goto bye;

4910

4911 /*

4912 * Select RSS Global Mode we want to use. We use "Basic Virtual"

4913 * mode which maps each Virtual Interface to its own section of

4914 * the RSS Table and we turn on all map and hash enables ...

4915 */

4916 adapter->flags |= RSS_TNLALLLOOKUP;

4917 ret = t4_config_glbl_rss(adapter, adapter->mbox,

4918 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,

4919 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |

4920 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ |

4921 ((adapter->flags & RSS_TNLALLLOOKUP) ?

4922 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP : 0));

4923 if (ret < 0)

4924 goto bye;

4925

4926 /*

4927 * Set up our own fundamental resource provisioning ...

4928 */

4929 ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,

4930 PFRES_NEQ, PFRES_NETHCTRL,

4931 PFRES_NIQFLINT, PFRES_NIQ,

4932 PFRES_TC, PFRES_NVI,

4933 FW_PFVF_CMD_CMASK_MASK,

4934 pfvfres_pmask(adapter, adapter->fn, 0),

4935 PFRES_NEXACTF,

4936 PFRES_R_CAPS, PFRES_WX_CAPS);

4937 if (ret < 0)

4938 goto bye;

4939

4940 /*

4941 * Perform low level SGE initialization. We need to do this before we

4942 * send the firmware the INITIALIZE command because that will cause

4943 * any other PF Drivers which are waiting for the Master

4944 * Initialization to proceed forward.

4945 */

4946 for (i = 0; i < SGE_NTIMERS - 1; i++)

4947 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);

4948 s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;

4949 s->counter_val[0] = 1;

4950 for (i = 1; i < SGE_NCOUNTERS; i++)

4951 s->counter_val[i] = min(intr_cnt[i - 1],

4952 THRESHOLD_0_GET(THRESHOLD_0_MASK));

4953 t4_sge_init(adapter);

4954

4955 #ifdef CONFIG_PCI_IOV

4956 /*

4957 * Provision resource limits for Virtual Functions. We currently

4958 * grant them all the same static resource limits except for the Port

4959 * Access Rights Mask which we're assigning based on the PF. All of

4960 * the static provisioning stuff for both the PF and VF really needs

4961 * to be managed in a persistent manner for each device which the

4962 * firmware controls.

4963 */

4964 {

4965 int pf, vf;

4966

4967 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {

4968 if (num_vf[pf] <= 0)

4969 continue;

4970

4971 /* VF numbering starts at 1! */

4972 for (vf = 1; vf <= num_vf[pf]; vf++) {

4973 ret = t4_cfg_pfvf(adapter, adapter->mbox,

4974 pf, vf,

4975 VFRES_NEQ, VFRES_NETHCTRL,

4976 VFRES_NIQFLINT, VFRES_NIQ,

4977 VFRES_TC, VFRES_NVI,

4978 FW_PFVF_CMD_CMASK_MASK,

4979 pfvfres_pmask(

4980 adapter, pf, vf),

4981 VFRES_NEXACTF,

4982 VFRES_R_CAPS, VFRES_WX_CAPS);

4983 if (ret < 0)

4984 dev_warn(adapter->pdev_dev,

4985 "failed to "\

4986 "provision pf/vf=%d/%d; "

4987 "err=%d\n", pf, vf, ret);

4988 }

4989 }

4990 }

4991 #endif

4992

4993 /*

4994 * Set up the default filter mode. Later we'll want to implement this

4995 * via a firmware command, etc. ... This needs to be done before the

4996 * firmare initialization command ... If the selected set of fields

4997 * isn't equal to the default value, we'll need to make sure that the

4998 * field selections will fit in the 36-bit budget.

4999 */

5000 if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {

5001 int j, bits = 0;

5002

5003 for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++)

5004 switch (tp_vlan_pri_map & (1 << j)) {

5005 case 0:

5006 /* compressed filter field not enabled */

5007 break;

5008 case FCOE_MASK:

5009 bits += 1;

5010 break;

5011 case PORT_MASK:

5012 bits += 3;

5013 break;

5014 case VNIC_ID_MASK:

5015 bits += 17;

5016 break;

5017 case VLAN_MASK:

5018 bits += 17;

5019 break;

5020 case TOS_MASK:

5021 bits += 8;

5022 break;

5023 case PROTOCOL_MASK:

5024 bits += 8;

5025 break;

5026 case ETHERTYPE_MASK:

5027 bits += 16;

5028 break;

5029 case MACMATCH_MASK:

5030 bits += 9;

5031 break;

5032 case MPSHITTYPE_MASK:

5033 bits += 3;

5034 break;

5035 case FRAGMENTATION_MASK:

5036 bits += 1;

5037 break;

5038 }

5039

5040 if (bits > 36) {

5041 dev_err(adapter->pdev_dev,

5042 "tp_vlan_pri_map=%#x needs %d bits > 36;"\

5043 " using %#x\n", tp_vlan_pri_map, bits,

5044 TP_VLAN_PRI_MAP_DEFAULT);

5045 tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;

5046 }

5047 }

5048 v = tp_vlan_pri_map;

5049 t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA,

5050 &v, 1, TP_VLAN_PRI_MAP);

5051

5052 /*

5053 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order

5054 * to support any of the compressed filter fields above. Newer

5055 * versions of the firmware do this automatically but it doesn't hurt

5056 * to set it here. Meanwhile, we do _not_ need to set Lookup Every

5057 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets

5058 * since the firmware automatically turns this on and off when we have

5059 * a non-zero number of filters active (since it does have a

5060 * performance impact).

5061 */

5062 if (tp_vlan_pri_map)

5063 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG,

5064 FIVETUPLELOOKUP_MASK,

5065 FIVETUPLELOOKUP_MASK);

5066

5067 /*

5068 * Tweak some settings.

5069 */

5070 t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) |

5071 RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |

5072 PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |

5073 KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));

5074

5075 /*

5076 * Get basic stuff going by issuing the Firmware Initialize command.

5077 * Note that this _must_ be after all PFVF commands ...

5078 */

5079 ret = t4_fw_initialize(adapter, adapter->mbox);

5080 if (ret < 0)

5081 goto bye;

5082

5083 /*

5084 * Return successfully!

5085 */

5086 dev_info(adapter->pdev_dev, "Successfully configured using built-in "\

5087 "driver parameters\n");

5088 return 0;

5089

5090 /*

5091 * Something bad happened. Return the error ...

5092 */

5093 bye:

5094 return ret;

5095 }

5096

5097 static struct fw_info fw_info_array[] = {

5098 {

5099 .chip = CHELSIO_T4,

5100 .fs_name = FW4_CFNAME,

5101 .fw_mod_name = FW4_FNAME,

5102 .fw_hdr = {

5103 .chip = FW_HDR_CHIP_T4,

5104 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),

5105 .intfver_nic = FW_INTFVER(T4, NIC),

5106 .intfver_vnic = FW_INTFVER(T4, VNIC),

5107 .intfver_ri = FW_INTFVER(T4, RI),

5108 .intfver_iscsi = FW_INTFVER(T4, ISCSI),

5109 .intfver_fcoe = FW_INTFVER(T4, FCOE),

5110 },

5111 }, {

5112 .chip = CHELSIO_T5,

5113 .fs_name = FW5_CFNAME,

5114 .fw_mod_name = FW5_FNAME,

5115 .fw_hdr = {

5116 .chip = FW_HDR_CHIP_T5,

5117 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),

5118 .intfver_nic = FW_INTFVER(T5, NIC),

5119 .intfver_vnic = FW_INTFVER(T5, VNIC),

5120 .intfver_ri = FW_INTFVER(T5, RI),

5121 .intfver_iscsi = FW_INTFVER(T5, ISCSI),

5122 .intfver_fcoe = FW_INTFVER(T5, FCOE),

5123 },

5124 }

5125 };

5126

5127 static struct fw_info *find_fw_info(int chip)

5128 {

5129 int i;

5130

5131 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {

5132 if (fw_info_array[i].chip == chip)

5133 return &fw_info_array[i];

5134 }

5135 return NULL;

5136 }

5137

5138 /*

5139 * Phase 0 of initialization: contact FW, obtain config, perform basic init.

5140 */

5141 static int adap_init0(struct adapter *adap)

5142 {

5143 int ret;

5144 u32 v, port_vec;

5145 enum dev_state state;

5146 u32 params[7], val[7];

5147 struct fw_caps_config_cmd caps_cmd;

5148 int reset = 1;

5149

5150 /*

5151 * Contact FW, advertising Master capability (and potentially forcing

5152 * ourselves as the Master PF if our module parameter force_init is

5153 * set).

5154 */

5155 ret = t4_fw_hello(adap, adap->mbox, adap->fn,

5156 force_init ? MASTER_MUST : MASTER_MAY,

5157 &state);

5158 if (ret < 0) {

5159 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",

5160 ret);

5161 return ret;

5162 }

5163 if (ret == adap->mbox)

5164 adap->flags |= MASTER_PF;

5165 if (force_init && state == DEV_STATE_INIT)

5166 state = DEV_STATE_UNINIT;

5167

5168 /*

5169 * If we're the Master PF Driver and the device is uninitialized,

5170 * then let's consider upgrading the firmware ... (We always want

5171 * to check the firmware version number in order to A. get it for

5172 * later reporting and B. to warn if the currently loaded firmware

5173 * is excessively mismatched relative to the driver.)

5174 */

5175 t4_get_fw_version(adap, &adap->params.fw_vers);

5176 t4_get_tp_version(adap, &adap->params.tp_vers);

5177 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {

5178 struct fw_info *fw_info;

5179 struct fw_hdr *card_fw;

5180 const struct firmware *fw;

5181 const u8 *fw_data = NULL;

5182 unsigned int fw_size = 0;

5183

5184 /* This is the firmware whose headers the driver was compiled

5185 * against

5186 */

5187 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));

5188 if (fw_info == NULL) {

5189 dev_err(adap->pdev_dev,

5190 "unable to get firmware info for chip %d.\n",

5191 CHELSIO_CHIP_VERSION(adap->params.chip));

5192 return -EINVAL;

5193 }

5194

5195 /* allocate memory to read the header of the firmware on the

5196 * card

5197 */

5198 card_fw = t4_alloc_mem(sizeof(*card_fw));

5199

5200 /* Get FW from from /lib/firmware/ */

5201 ret = request_firmware(&fw, fw_info->fw_mod_name,

5202 adap->pdev_dev);

5203 if (ret < 0) {

5204 dev_err(adap->pdev_dev,

5205 "unable to load firmware image %s, error %d\n",

5206 fw_info->fw_mod_name, ret);

5207 } else {

5208 fw_data = fw->data;

5209 fw_size = fw->size;

5210 }

5211

5212 /* upgrade FW logic */

5213 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,

5214 state, &reset);

5215

5216 /* Cleaning up */

5217 if (fw != NULL)

5218 release_firmware(fw);

5219 t4_free_mem(card_fw);

5220

5221 if (ret < 0)

5222 goto bye;

5223 }

5224

5225 /*

5226 * Grab VPD parameters. This should be done after we establish a

5227 * connection to the firmware since some of the VPD parameters

5228 * (notably the Core Clock frequency) are retrieved via requests to

5229 * the firmware. On the other hand, we need these fairly early on

5230 * so we do this right after getting ahold of the firmware.

5231 */

5232 ret = get_vpd_params(adap, &adap->params.vpd);

5233 if (ret < 0)

5234 goto bye;

5235

5236 /*

5237 * Find out what ports are available to us. Note that we need to do

5238 * this before calling adap_init0_no_config() since it needs nports

5239 * and portvec ...

5240 */

5241 v =

5242 FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |

5243 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);

5244 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);

5245 if (ret < 0)

5246 goto bye;

5247

5248 adap->params.nports = hweight32(port_vec);

5249 adap->params.portvec = port_vec;

5250

5251 /*

5252 * If the firmware is initialized already (and we're not forcing a

5253 * master initialization), note that we're living with existing

5254 * adapter parameters. Otherwise, it's time to try initializing the

5255 * adapter ...

5256 */

5257 if (state == DEV_STATE_INIT) {

5258 dev_info(adap->pdev_dev, "Coming up as %s: "\

5259 "Adapter already initialized\n",

5260 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");

5261 adap->flags |= USING_SOFT_PARAMS;

5262 } else {

5263 dev_info(adap->pdev_dev, "Coming up as MASTER: "\

5264 "Initializing adapter\n");

5265

5266 /*

5267 * If the firmware doesn't support Configuration

5268 * Files warn user and exit,

5269 */

5270 if (ret < 0)

5271 dev_warn(adap->pdev_dev, "Firmware doesn't support "

5272 "configuration file.\n");

5273 if (force_old_init)

5274 ret = adap_init0_no_config(adap, reset);

5275 else {

5276 /*

5277 * Find out whether we're dealing with a version of

5278 * the firmware which has configuration file support.

5279 */

5280 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |

5281 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));

5282 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,

5283 params, val);

5284

5285 /*

5286 * If the firmware doesn't support Configuration

5287 * Files, use the old Driver-based, hard-wired

5288 * initialization. Otherwise, try using the

5289 * Configuration File support and fall back to the

5290 * Driver-based initialization if there's no

5291 * Configuration File found.

5292 */

5293 if (ret < 0)

5294 ret = adap_init0_no_config(adap, reset);

5295 else {

5296 /*

5297 * The firmware provides us with a memory

5298 * buffer where we can load a Configuration

5299 * File from the host if we want to override

5300 * the Configuration File in flash.

5301 */

5302

5303 ret = adap_init0_config(adap, reset);

5304 if (ret == -ENOENT) {

5305 dev_info(adap->pdev_dev,

5306 "No Configuration File present "

5307 "on adapter. Using hard-wired "

5308 "configuration parameters.\n");

5309 ret = adap_init0_no_config(adap, reset);

5310 }

5311 }

5312 }

5313 if (ret < 0) {

5314 dev_err(adap->pdev_dev,

5315 "could not initialize adapter, error %d\n",

5316 -ret);

5317 goto bye;

5318 }

5319 }

5320

5321 /*

5322 * If we're living with non-hard-coded parameters (either from a

5323 * Firmware Configuration File or values programmed by a different PF

5324 * Driver), give the SGE code a chance to pull in anything that it

5325 * needs ... Note that this must be called after we retrieve our VPD

5326 * parameters in order to know how to convert core ticks to seconds.

5327 */

5328 if (adap->flags & USING_SOFT_PARAMS) {

5329 ret = t4_sge_init(adap);

5330 if (ret < 0)

5331 goto bye;

5332 }

5333

5334 if (is_bypass_device(adap->pdev->device))

5335 adap->params.bypass = 1;

5336

5337 /*

5338 * Grab some of our basic fundamental operating parameters.

5339 */

5340 #define FW_PARAM_DEV(param) \

5341 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \

5342 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))

5343

5344 #define FW_PARAM_PFVF(param) \

5345 FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \

5346 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)| \

5347 FW_PARAMS_PARAM_Y(0) | \

5348 FW_PARAMS_PARAM_Z(0)

5349

5350 params[0] = FW_PARAM_PFVF(EQ_START);

5351 params[1] = FW_PARAM_PFVF(L2T_START);

5352 params[2] = FW_PARAM_PFVF(L2T_END);

5353 params[3] = FW_PARAM_PFVF(FILTER_START);

5354 params[4] = FW_PARAM_PFVF(FILTER_END);

5355 params[5] = FW_PARAM_PFVF(IQFLINT_START);

5356 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);

5357 if (ret < 0)

5358 goto bye;

5359 adap->sge.egr_start = val[0];

5360 adap->l2t_start = val[1];

5361 adap->l2t_end = val[2];

5362 adap->tids.ftid_base = val[3];

5363 adap->tids.nftids = val[4] - val[3] + 1;

5364 adap->sge.ingr_start = val[5];

5365

5366 /* query params related to active filter region */

5367 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);

5368 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);

5369 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);

5370 /* If Active filter size is set we enable establishing

5371 * offload connection through firmware work request

5372 */

5373 if ((val[0] != val[1]) && (ret >= 0)) {

5374 adap->flags |= FW_OFLD_CONN;

5375 adap->tids.aftid_base = val[0];

5376 adap->tids.aftid_end = val[1];

5377 }

5378

5379 /* If we're running on newer firmware, let it know that we're

5380 * prepared to deal with encapsulated CPL messages. Older

5381 * firmware won't understand this and we'll just get

5382 * unencapsulated messages ...

5383 */

5384 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);

5385 val[0] = 1;

5386 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);

5387

5388 /*

5389 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL

5390 * capability. Earlier versions of the firmware didn't have the

5391 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no

5392 * permission to use ULPTX MEMWRITE DSGL.

5393 */

5394 if (is_t4(adap->params.chip)) {

5395 adap->params.ulptx_memwrite_dsgl = false;

5396 } else {

5397 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);

5398 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,

5399 1, params, val);

5400 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);

5401 }

5402

5403 /*

5404 * Get device capabilities so we can determine what resources we need

5405 * to manage.

5406 */

5407 memset(&caps_cmd, 0, sizeof(caps_cmd));

5408 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |

5409 FW_CMD_REQUEST | FW_CMD_READ);

5410 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));

5411 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),

5412 &caps_cmd);

5413 if (ret < 0)

5414 goto bye;

5415

5416 if (caps_cmd.ofldcaps) {

5417 /* query offload-related parameters */

5418 params[0] = FW_PARAM_DEV(NTID);

5419 params[1] = FW_PARAM_PFVF(SERVER_START);

5420 params[2] = FW_PARAM_PFVF(SERVER_END);

5421 params[3] = FW_PARAM_PFVF(TDDP_START);

5422 params[4] = FW_PARAM_PFVF(TDDP_END);

5423 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);

5424 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,

5425 params, val);

5426 if (ret < 0)

5427 goto bye;

5428 adap->tids.ntids = val[0];

5429 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);

5430 adap->tids.stid_base = val[1];

5431 adap->tids.nstids = val[2] - val[1] + 1;

5432 /*

5433 * Setup server filter region. Divide the availble filter

5434 * region into two parts. Regular filters get 1/3rd and server

5435 * filters get 2/3rd part. This is only enabled if workarond

5436 * path is enabled.

5437 * 1. For regular filters.

5438 * 2. Server filter: This are special filters which are used

5439 * to redirect SYN packets to offload queue.

5440 */

5441 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {

5442 adap->tids.sftid_base = adap->tids.ftid_base +

5443 DIV_ROUND_UP(adap->tids.nftids, 3);

5444 adap->tids.nsftids = adap->tids.nftids -

5445 DIV_ROUND_UP(adap->tids.nftids, 3);

5446 adap->tids.nftids = adap->tids.sftid_base -

5447 adap->tids.ftid_base;

5448 }

5449 adap->vres.ddp.start = val[3];

5450 adap->vres.ddp.size = val[4] - val[3] + 1;

5451 adap->params.ofldq_wr_cred = val[5];

5452

5453 adap->params.offload = 1;

5454 }

5455 if (caps_cmd.rdmacaps) {

5456 params[0] = FW_PARAM_PFVF(STAG_START);

5457 params[1] = FW_PARAM_PFVF(STAG_END);

5458 params[2] = FW_PARAM_PFVF(RQ_START);

5459 params[3] = FW_PARAM_PFVF(RQ_END);

5460 params[4] = FW_PARAM_PFVF(PBL_START);

5461 params[5] = FW_PARAM_PFVF(PBL_END);

5462 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,

5463 params, val);

5464 if (ret < 0)

5465 goto bye;

5466 adap->vres.stag.start = val[0];

5467 adap->vres.stag.size = val[1] - val[0] + 1;

5468 adap->vres.rq.start = val[2];

5469 adap->vres.rq.size = val[3] - val[2] + 1;

5470 adap->vres.pbl.start = val[4];

5471 adap->vres.pbl.size = val[5] - val[4] + 1;

5472

5473 params[0] = FW_PARAM_PFVF(SQRQ_START);

5474 params[1] = FW_PARAM_PFVF(SQRQ_END);

5475 params[2] = FW_PARAM_PFVF(CQ_START);

5476 params[3] = FW_PARAM_PFVF(CQ_END);

5477 params[4] = FW_PARAM_PFVF(OCQ_START);

5478 params[5] = FW_PARAM_PFVF(OCQ_END);

5479 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);

5480 if (ret < 0)

5481 goto bye;

5482 adap->vres.qp.start = val[0];

5483 adap->vres.qp.size = val[1] - val[0] + 1;

5484 adap->vres.cq.start = val[2];

5485 adap->vres.cq.size = val[3] - val[2] + 1;

5486 adap->vres.ocq.start = val[4];

5487 adap->vres.ocq.size = val[5] - val[4] + 1;

5488 }

5489 if (caps_cmd.iscsicaps) {

5490 params[0] = FW_PARAM_PFVF(ISCSI_START);

5491 params[1] = FW_PARAM_PFVF(ISCSI_END);

5492 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,

5493 params, val);

5494 if (ret < 0)

5495 goto bye;

5496 adap->vres.iscsi.start = val[0];

5497 adap->vres.iscsi.size = val[1] - val[0] + 1;

5498 }

5499 #undef FW_PARAM_PFVF

5500 #undef FW_PARAM_DEV

5501

5502 /*

5503 * These are finalized by FW initialization, load their values now.

5504 */

5505 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);

5506 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,

5507 adap->params.b_wnd);

5508

5509 t4_init_tp_params(adap);

5510 adap->flags |= FW_OK;

5511 return 0;

5512

5513 /*

5514 * Something bad happened. If a command timed out or failed with EIO

5515 * FW does not operate within its spec or something catastrophic

5516 * happened to HW/FW, stop issuing commands.

5517 */

5518 bye:

5519 if (ret != -ETIMEDOUT && ret != -EIO)

5520 t4_fw_bye(adap, adap->mbox);

5521 return ret;

5522 }

5523

5524 /* EEH callbacks */

5525

5526 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,

5527 pci_channel_state_t state)

5528 {

5529 int i;

5530 struct adapter *adap = pci_get_drvdata(pdev);

5531

5532 if (!adap)

5533 goto out;

5534

5535 rtnl_lock();

5536 adap->flags &= ~FW_OK;

5537 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);

5538 spin_lock(&adap->stats_lock);

5539 for_each_port(adap, i) {

5540 struct net_device *dev = adap->port[i];

5541

5542 netif_device_detach(dev);

5543 netif_carrier_off(dev);

5544 }

5545 spin_unlock(&adap->stats_lock);

5546 if (adap->flags & FULL_INIT_DONE)

5547 cxgb_down(adap);

5548 rtnl_unlock();

5549 if ((adap->flags & DEV_ENABLED)) {

5550 pci_disable_device(pdev);

5551 adap->flags &= ~DEV_ENABLED;

5552 }

5553 out: return state == pci_channel_io_perm_failure ?

5554 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;

5555 }

5556

5557 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)

5558 {

5559 int i, ret;

5560 struct fw_caps_config_cmd c;

5561 struct adapter *adap = pci_get_drvdata(pdev);

5562

5563 if (!adap) {

5564 pci_restore_state(pdev);

5565 pci_save_state(pdev);

5566 return PCI_ERS_RESULT_RECOVERED;

5567 }

5568

5569 if (!(adap->flags & DEV_ENABLED)) {

5570 if (pci_enable_device(pdev)) {

5571 dev_err(&pdev->dev, "Cannot reenable PCI "

5572 "device after reset\n");

5573 return PCI_ERS_RESULT_DISCONNECT;

5574 }

5575 adap->flags |= DEV_ENABLED;

5576 }

5577

5578 pci_set_master(pdev);

5579 pci_restore_state(pdev);

5580 pci_save_state(pdev);

5581 pci_cleanup_aer_uncorrect_error_status(pdev);

5582

5583 if (t4_wait_dev_ready(adap) < 0)

5584 return PCI_ERS_RESULT_DISCONNECT;

5585 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)

5586 return PCI_ERS_RESULT_DISCONNECT;

5587 adap->flags |= FW_OK;

5588 if (adap_init1(adap, &c))

5589 return PCI_ERS_RESULT_DISCONNECT;

5590

5591 for_each_port(adap, i) {

5592 struct port_info *p = adap2pinfo(adap, i);

5593

5594 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,

5595 NULL, NULL);

5596 if (ret < 0)

5597 return PCI_ERS_RESULT_DISCONNECT;

5598 p->viid = ret;

5599 p->xact_addr_filt = -1;

5600 }

5601

5602 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,

5603 adap->params.b_wnd);

5604 setup_memwin(adap);

5605 if (cxgb_up(adap))

5606 return PCI_ERS_RESULT_DISCONNECT;

5607 return PCI_ERS_RESULT_RECOVERED;

5608 }

5609

5610 static void eeh_resume(struct pci_dev *pdev)

5611 {

5612 int i;

5613 struct adapter *adap = pci_get_drvdata(pdev);

5614

5615 if (!adap)

5616 return;

5617

5618 rtnl_lock();

5619 for_each_port(adap, i) {

5620 struct net_device *dev = adap->port[i];

5621

5622 if (netif_running(dev)) {

5623 link_start(dev);

5624 cxgb_set_rxmode(dev);

5625 }

5626 netif_device_attach(dev);

5627 }

5628 rtnl_unlock();

5629 }

5630

5631 static const struct pci_error_handlers cxgb4_eeh = {

5632 .error_detected = eeh_err_detected,

5633 .slot_reset = eeh_slot_reset,

5634 .resume = eeh_resume,

5635 };

5636

5637 static inline bool is_x_10g_port(const struct link_config *lc)

5638 {

5639 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||

5640 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;

5641 }

5642

5643 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,

5644 unsigned int size, unsigned int iqe_size)

5645 {

5646 q->intr_params = QINTR_TIMER_IDX(timer_idx) |

5647 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);

5648 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;

5649 q->iqe_len = iqe_size;

5650 q->size = size;

5651 }

5652

5653 /*

5654 * Perform default configuration of DMA queues depending on the number and type

5655 * of ports we found and the number of available CPUs. Most settings can be

5656 * modified by the admin prior to actual use.

5657 */

5658 static void cfg_queues(struct adapter *adap)

5659 {

5660 struct sge *s = &adap->sge;

5661 int i, q10g = 0, n10g = 0, qidx = 0;

5662

5663 for_each_port(adap, i)

5664 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);

5665

5666 /*

5667 * We default to 1 queue per non-10G port and up to # of cores queues

5668 * per 10G port.

5669 */

5670 if (n10g)

5671 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;

5672 if (q10g > netif_get_num_default_rss_queues())

5673 q10g = netif_get_num_default_rss_queues();

5674

5675 for_each_port(adap, i) {

5676 struct port_info *pi = adap2pinfo(adap, i);

5677

5678 pi->first_qset = qidx;

5679 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;

5680 qidx += pi->nqsets;

5681 }

5682

5683 s->ethqsets = qidx;

5684 s->max_ethqsets = qidx; /* MSI-X may lower it later */

5685

5686 if (is_offload(adap)) {

5687 /*

5688 * For offload we use 1 queue/channel if all ports are up to 1G,

5689 * otherwise we divide all available queues amongst the channels

5690 * capped by the number of available cores.

5691 */

5692 if (n10g) {

5693 i = min_t(int, ARRAY_SIZE(s->ofldrxq),

5694 num_online_cpus());

5695 s->ofldqsets = roundup(i, adap->params.nports);

5696 } else

5697 s->ofldqsets = adap->params.nports;

5698 /* For RDMA one Rx queue per channel suffices */

5699 s->rdmaqs = adap->params.nports;

5700 }

5701

5702 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {

5703 struct sge_eth_rxq *r = &s->ethrxq[i];

5704

5705 init_rspq(&r->rspq, 0, 0, 1024, 64);

5706 r->fl.size = 72;

5707 }

5708

5709 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)

5710 s->ethtxq[i].q.size = 1024;

5711

5712 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)

5713 s->ctrlq[i].q.size = 512;

5714

5715 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)

5716 s->ofldtxq[i].q.size = 1024;

5717

5718 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {

5719 struct sge_ofld_rxq *r = &s->ofldrxq[i];

5720

5721 init_rspq(&r->rspq, 0, 0, 1024, 64);

5722 r->rspq.uld = CXGB4_ULD_ISCSI;

5723 r->fl.size = 72;

5724 }

5725

5726 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {

5727 struct sge_ofld_rxq *r = &s->rdmarxq[i];

5728

5729 init_rspq(&r->rspq, 0, 0, 511, 64);

5730 r->rspq.uld = CXGB4_ULD_RDMA;

5731 r->fl.size = 72;

5732 }

5733

5734 init_rspq(&s->fw_evtq, 6, 0, 512, 64);

5735 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);

5736 }

5737

5738 /*

5739 * Reduce the number of Ethernet queues across all ports to at most n.

5740 * n provides at least one queue per port.

5741 */

5742 static void reduce_ethqs(struct adapter *adap, int n)

5743 {

5744 int i;

5745 struct port_info *pi;

5746

5747 while (n < adap->sge.ethqsets)

5748 for_each_port(adap, i) {

5749 pi = adap2pinfo(adap, i);

5750 if (pi->nqsets > 1) {

5751 pi->nqsets--;

5752 adap->sge.ethqsets--;

5753 if (adap->sge.ethqsets <= n)

5754 break;

5755 }

5756 }

5757

5758 n = 0;

5759 for_each_port(adap, i) {

5760 pi = adap2pinfo(adap, i);

5761 pi->first_qset = n;

5762 n += pi->nqsets;

5763 }

5764 }

5765

5766 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */

5767 #define EXTRA_VECS 2

5768

5769 static int enable_msix(struct adapter *adap)

5770 {

5771 int ofld_need = 0;

5772 int i, want, need;

5773 struct sge *s = &adap->sge;

5774 unsigned int nchan = adap->params.nports;

5775 struct msix_entry entries[MAX_INGQ + 1];

5776

5777 for (i = 0; i < ARRAY_SIZE(entries); ++i)

5778 entries[i].entry = i;

5779

5780 want = s->max_ethqsets + EXTRA_VECS;

5781 if (is_offload(adap)) {

5782 want += s->rdmaqs + s->ofldqsets;

5783 /* need nchan for each possible ULD */

5784 ofld_need = 2 * nchan;

5785 }

5786 need = adap->params.nports + EXTRA_VECS + ofld_need;

5787

5788 want = pci_enable_msix_range(adap->pdev, entries, need, want);

5789 if (want < 0)

5790 return want;

5791

5792 /*

5793 * Distribute available vectors to the various queue groups.

5794 * Every group gets its minimum requirement and NIC gets top

5795 * priority for leftovers.

5796 */

5797 i = want - EXTRA_VECS - ofld_need;

5798 if (i < s->max_ethqsets) {

5799 s->max_ethqsets = i;

5800 if (i < s->ethqsets)

5801 reduce_ethqs(adap, i);

5802 }

5803 if (is_offload(adap)) {

5804 i = want - EXTRA_VECS - s->max_ethqsets;

5805 i -= ofld_need - nchan;

5806 s->ofldqsets = (i / nchan) * nchan; /* round down */

5807 }

5808 for (i = 0; i < want; ++i)

5809 adap->msix_info[i].vec = entries[i].vector;

5810

5811 return 0;

5812 }

5813

5814 #undef EXTRA_VECS

5815

5816 static int init_rss(struct adapter *adap)

5817 {

5818 unsigned int i, j;

5819

5820 for_each_port(adap, i) {

5821 struct port_info *pi = adap2pinfo(adap, i);

5822

5823 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);

5824 if (!pi->rss)

5825 return -ENOMEM;

5826 for (j = 0; j < pi->rss_size; j++)

5827 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);

5828 }

5829 return 0;

5830 }

5831

5832 static void print_port_info(const struct net_device *dev)

5833 {

5834 char buf[80];

5835 char *bufp = buf;

5836 const char *spd = "";

5837 const struct port_info *pi = netdev_priv(dev);

5838 const struct adapter *adap = pi->adapter;

5839

5840 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)

5841 spd = " 2.5 GT/s";

5842 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)

5843 spd = " 5 GT/s";

5844

5845 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)

5846 bufp += sprintf(bufp, "100/");

5847 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)

5848 bufp += sprintf(bufp, "1000/");

5849 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)

5850 bufp += sprintf(bufp, "10G/");

5851 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)

5852 bufp += sprintf(bufp, "40G/");

5853 if (bufp != buf)

5854 --bufp;

5855 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));

5856

5857 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",

5858 adap->params.vpd.id,

5859 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,

5860 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,

5861 (adap->flags & USING_MSIX) ? " MSI-X" :

5862 (adap->flags & USING_MSI) ? " MSI" : "");

5863 netdev_info(dev, "S/N: %s, P/N: %s\n",

5864 adap->params.vpd.sn, adap->params.vpd.pn);

5865 }

5866

5867 static void enable_pcie_relaxed_ordering(struct pci_dev *dev)

5868 {

5869 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);

5870 }

5871

5872 /*

5873 * Free the following resources:

5874 * - memory used for tables

5875 * - MSI/MSI-X

5876 * - net devices

5877 * - resources FW is holding for us

5878 */

5879 static void free_some_resources(struct adapter *adapter)

5880 {

5881 unsigned int i;

5882

5883 t4_free_mem(adapter->l2t);

5884 t4_free_mem(adapter->tids.tid_tab);

5885 disable_msi(adapter);

5886

5887 for_each_port(adapter, i)

5888 if (adapter->port[i]) {

5889 kfree(adap2pinfo(adapter, i)->rss);

5890 free_netdev(adapter->port[i]);

5891 }

5892 if (adapter->flags & FW_OK)

5893 t4_fw_bye(adapter, adapter->fn);

5894 }

5895

5896 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)

5897 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \

5898 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)

5899 #define SEGMENT_SIZE 128

5900

5901 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)

5902 {

5903 int func, i, err, s_qpp, qpp, num_seg;

5904 struct port_info *pi;

5905 bool highdma = false;

5906 struct adapter *adapter = NULL;

5907

5908 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);

5909

5910 err = pci_request_regions(pdev, KBUILD_MODNAME);

5911 if (err) {

5912 /* Just info, some other driver may have claimed the device. */

5913 dev_info(&pdev->dev, "cannot obtain PCI resources\n");

5914 return err;

5915 }

5916

5917 /* We control everything through one PF */

5918 func = PCI_FUNC(pdev->devfn);

5919 if (func != ent->driver_data) {

5920 pci_save_state(pdev); /* to restore SR-IOV later */

5921 goto sriov;

5922 }

5923

5924 err = pci_enable_device(pdev);

5925 if (err) {

5926 dev_err(&pdev->dev, "cannot enable PCI device\n");

5927 goto out_release_regions;

5928 }

5929

5930 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {

5931 highdma = true;

5932 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));

5933 if (err) {

5934 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "

5935 "coherent allocations\n");

5936 goto out_disable_device;

5937 }

5938 } else {

5939 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));

5940 if (err) {

5941 dev_err(&pdev->dev, "no usable DMA configuration\n");

5942 goto out_disable_device;

5943 }

5944 }

5945

5946 pci_enable_pcie_error_reporting(pdev);

5947 enable_pcie_relaxed_ordering(pdev);

5948 pci_set_master(pdev);

5949 pci_save_state(pdev);

5950

5951 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);

5952 if (!adapter) {

5953 err = -ENOMEM;

5954 goto out_disable_device;

5955 }

5956

5957 /* PCI device has been enabled */

5958 adapter->flags |= DEV_ENABLED;

5959

5960 adapter->regs = pci_ioremap_bar(pdev, 0);

5961 if (!adapter->regs) {

5962 dev_err(&pdev->dev, "cannot map device registers\n");

5963 err = -ENOMEM;

5964 goto out_free_adapter;

5965 }

5966

5967 adapter->pdev = pdev;

5968 adapter->pdev_dev = &pdev->dev;

5969 adapter->mbox = func;

5970 adapter->fn = func;

5971 adapter->msg_enable = dflt_msg_enable;

5972 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));

5973

5974 spin_lock_init(&adapter->stats_lock);

5975 spin_lock_init(&adapter->tid_release_lock);

5976

5977 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);

5978 INIT_WORK(&adapter->db_full_task, process_db_full);

5979 INIT_WORK(&adapter->db_drop_task, process_db_drop);

5980

5981 err = t4_prep_adapter(adapter);

5982 if (err)

5983 goto out_unmap_bar0;

5984

5985 if (!is_t4(adapter->params.chip)) {

5986 s_qpp = QUEUESPERPAGEPF1 * adapter->fn;

5987 qpp = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter,

5988 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);

5989 num_seg = PAGE_SIZE / SEGMENT_SIZE;

5990

5991 /* Each segment size is 128B. Write coalescing is enabled only

5992 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the

5993 * queue is less no of segments that can be accommodated in

5994 * a page size.

5995 */

5996 if (qpp > num_seg) {

5997 dev_err(&pdev->dev,

5998 "Incorrect number of egress queues per page\n");

5999 err = -EINVAL;

6000 goto out_unmap_bar0;

6001 }

6002 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),

6003 pci_resource_len(pdev, 2));

6004 if (!adapter->bar2) {

6005 dev_err(&pdev->dev, "cannot map device bar2 region\n");

6006 err = -ENOMEM;

6007 goto out_unmap_bar0;

6008 }

6009 }

6010

6011 setup_memwin(adapter);

6012 err = adap_init0(adapter);

6013 setup_memwin_rdma(adapter);

6014 if (err)

6015 goto out_unmap_bar;

6016

6017 for_each_port(adapter, i) {

6018 struct net_device *netdev;

6019

6020 netdev = alloc_etherdev_mq(sizeof(struct port_info),

6021 MAX_ETH_QSETS);

6022 if (!netdev) {

6023 err = -ENOMEM;

6024 goto out_free_dev;

6025 }

6026

6027 SET_NETDEV_DEV(netdev, &pdev->dev);

6028

6029 adapter->port[i] = netdev;

6030 pi = netdev_priv(netdev);

6031 pi->adapter = adapter;

6032 pi->xact_addr_filt = -1;

6033 pi->port_id = i;

6034 netdev->irq = pdev->irq;

6035

6036 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |

6037 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |

6038 NETIF_F_RXCSUM | NETIF_F_RXHASH |

6039 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;

6040 if (highdma)

6041 netdev->hw_features |= NETIF_F_HIGHDMA;

6042 netdev->features |= netdev->hw_features;

6043 netdev->vlan_features = netdev->features & VLAN_FEAT;

6044

6045 netdev->priv_flags |= IFF_UNICAST_FLT;

6046

6047 netdev->netdev_ops = &cxgb4_netdev_ops;

6048 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);

6049 }

6050

6051 pci_set_drvdata(pdev, adapter);

6052

6053 if (adapter->flags & FW_OK) {

6054 err = t4_port_init(adapter, func, func, 0);

6055 if (err)

6056 goto out_free_dev;

6057 }

6058

6059 /*

6060 * Configure queues and allocate tables now, they can be needed as

6061 * soon as the first register_netdev completes.

6062 */

6063 cfg_queues(adapter);

6064

6065 adapter->l2t = t4_init_l2t();

6066 if (!adapter->l2t) {

6067 /* We tolerate a lack of L2T, giving up some functionality */

6068 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");

6069 adapter->params.offload = 0;

6070 }

6071

6072 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {

6073 dev_warn(&pdev->dev, "could not allocate TID table, "

6074 "continuing\n");

6075 adapter->params.offload = 0;

6076 }

6077

6078 /* See what interrupts we'll be using */

6079 if (msi > 1 && enable_msix(adapter) == 0)

6080 adapter->flags |= USING_MSIX;

6081 else if (msi > 0 && pci_enable_msi(pdev) == 0)

6082 adapter->flags |= USING_MSI;

6083

6084 err = init_rss(adapter);

6085 if (err)

6086 goto out_free_dev;

6087

6088 /*

6089 * The card is now ready to go. If any errors occur during device

6090 * registration we do not fail the whole card but rather proceed only

6091 * with the ports we manage to register successfully. However we must

6092 * register at least one net device.

6093 */

6094 for_each_port(adapter, i) {

6095 pi = adap2pinfo(adapter, i);

6096 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);

6097 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);

6098

6099 err = register_netdev(adapter->port[i]);

6100 if (err)

6101 break;

6102 adapter->chan_map[pi->tx_chan] = i;

6103 print_port_info(adapter->port[i]);

6104 }

6105 if (i == 0) {

6106 dev_err(&pdev->dev, "could not register any net devices\n");

6107 goto out_free_dev;

6108 }

6109 if (err) {

6110 dev_warn(&pdev->dev, "only %d net devices registered\n", i);

6111 err = 0;

6112 }

6113

6114 if (cxgb4_debugfs_root) {

6115 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),

6116 cxgb4_debugfs_root);

6117 setup_debugfs(adapter);

6118 }

6119

6120 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */

6121 pdev->needs_freset = 1;

6122

6123 if (is_offload(adapter))

6124 attach_ulds(adapter);

6125

6126 sriov:

6127 #ifdef CONFIG_PCI_IOV

6128 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)

6129 if (pci_enable_sriov(pdev, num_vf[func]) == 0)

6130 dev_info(&pdev->dev,

6131 "instantiated %u virtual functions\n",

6132 num_vf[func]);

6133 #endif

6134 return 0;

6135

6136 out_free_dev:

6137 free_some_resources(adapter);

6138 out_unmap_bar:

6139 if (!is_t4(adapter->params.chip))

6140 iounmap(adapter->bar2);

6141 out_unmap_bar0:

6142 iounmap(adapter->regs);

6143 out_free_adapter:

6144 kfree(adapter);

6145 out_disable_device:

6146 pci_disable_pcie_error_reporting(pdev);

6147 pci_disable_device(pdev);

6148 out_release_regions:

6149 pci_release_regions(pdev);

6150 return err;

6151 }

6152

6153 static void remove_one(struct pci_dev *pdev)

6154 {

6155 struct adapter *adapter = pci_get_drvdata(pdev);

6156

6157 #ifdef CONFIG_PCI_IOV

6158 pci_disable_sriov(pdev);

6159

6160 #endif

6161

6162 if (adapter) {

6163 int i;

6164

6165 if (is_offload(adapter))

6166 detach_ulds(adapter);

6167

6168 for_each_port(adapter, i)

6169 if (adapter->port[i]->reg_state == NETREG_REGISTERED)

6170 unregister_netdev(adapter->port[i]);

6171

6172 if (adapter->debugfs_root)

6173 debugfs_remove_recursive(adapter->debugfs_root);

6174

6175 /* If we allocated filters, free up state associated with any

6176 * valid filters ...

6177 */

6178 if (adapter->tids.ftid_tab) {

6179 struct filter_entry *f = &adapter->tids.ftid_tab[0];

6180 for (i = 0; i < (adapter->tids.nftids +

6181 adapter->tids.nsftids); i++, f++)

6182 if (f->valid)

6183 clear_filter(adapter, f);

6184 }

6185

6186 if (adapter->flags & FULL_INIT_DONE)

6187 cxgb_down(adapter);

6188

6189 free_some_resources(adapter);

6190 iounmap(adapter->regs);

6191 if (!is_t4(adapter->params.chip))

6192 iounmap(adapter->bar2);

6193 pci_disable_pcie_error_reporting(pdev);

6194 if ((adapter->flags & DEV_ENABLED)) {

6195 pci_disable_device(pdev);

6196 adapter->flags &= ~DEV_ENABLED;

6197 }

6198 pci_release_regions(pdev);

6199 kfree(adapter);

6200 } else

6201 pci_release_regions(pdev);

6202 }

6203

6204 static struct pci_driver cxgb4_driver = {

6205 .name = KBUILD_MODNAME,

6206 .id_table = cxgb4_pci_tbl,

6207 .probe = init_one,

6208 .remove = remove_one,

6209 .err_handler = &cxgb4_eeh,

6210 };

6211

6212 static int __init cxgb4_init_module(void)

6213 {

6214 int ret;

6215

6216 workq = create_singlethread_workqueue("cxgb4");

6217 if (!workq)

6218 return -ENOMEM;

6219

6220 /* Debugfs support is optional, just warn if this fails */

6221 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);

6222 if (!cxgb4_debugfs_root)

6223 pr_warn("could not create debugfs entry, continuing\n");

6224

6225 ret = pci_register_driver(&cxgb4_driver);

6226 if (ret < 0) {

6227 debugfs_remove(cxgb4_debugfs_root);

6228 destroy_workqueue(workq);

6229 }

6230

6231 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);

6232

6233 return ret;

6234 }

6235

6236 static void __exit cxgb4_cleanup_module(void)

6237 {

6238 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);

6239 pci_unregister_driver(&cxgb4_driver);

6240 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */

6241 flush_workqueue(workq);

6242 destroy_workqueue(workq);

6243 }

6244

6245 module_init(cxgb4_init_module);

6246 module_exit(cxgb4_cleanup_module);