diff --git a/usr/src/uts/common/io/mlxcx/mlxcx.c b/usr/src/uts/common/io/mlxcx/mlxcx.c index f74d093b9c04..bd4c08919fde 100644 --- a/usr/src/uts/common/io/mlxcx/mlxcx.c +++ b/usr/src/uts/common/io/mlxcx/mlxcx.c @@ -13,6 +13,7 @@ * Copyright 2021, The University of Queensland * Copyright (c) 2018, Joyent, Inc. * Copyright 2020 RackTop Systems, Inc. + * Copyright 2023 MNX Cloud, Inc. */ /* @@ -20,9 +21,15 @@ */ /* - * The PRM for this family of parts is freely available, and can be found at: - * https://www.mellanox.com/related-docs/user_manuals/ \ - * Ethernet_Adapters_Programming_Manual.pdf + * The PRM for this family of parts was freely available at: + * + * BEGIN CSTYLED + * https://www.mellanox.com/related-docs/user_manuals/Ethernet_Adapters_Programming_Manual.pdf + * END CSTYLED + * + * But has since disappeared. The FreeBSD driver has some visibility into how + * things still work, but there have been changes in the 6-series that we are + * only beginning to adopt here. */ /* * ConnectX glossary @@ -481,12 +488,17 @@ mlxcx_load_prop_defaults(mlxcx_t *mlxp) * maximum speed of 10Gb/s, and another for those above that. */ if ((port->mlp_max_proto & (MLXCX_PROTO_25G | MLXCX_PROTO_40G | - MLXCX_PROTO_50G | MLXCX_PROTO_100G)) != 0) { + MLXCX_PROTO_50G | MLXCX_PROTO_100G)) != 0 || + (port->mlp_ext_max_proto & (MLXCX_EXTPROTO_25G | + MLXCX_EXTPROTO_40G | MLXCX_EXTPROTO_50G | + MLXCX_EXTPROTO_100G)) != 0) { p->mldp_cq_size_shift_default = MLXCX_CQ_SIZE_SHIFT_25G; p->mldp_rq_size_shift_default = MLXCX_RQ_SIZE_SHIFT_25G; p->mldp_sq_size_shift_default = MLXCX_SQ_SIZE_SHIFT_25G; } else if ((port->mlp_max_proto & (MLXCX_PROTO_100M | MLXCX_PROTO_1G | - MLXCX_PROTO_10G)) != 0) { + MLXCX_PROTO_10G)) != 0 || + (port->mlp_ext_max_proto & (MLXCX_EXTPROTO_100M | + MLXCX_EXTPROTO_5G | MLXCX_EXTPROTO_1G | MLXCX_EXTPROTO_10G)) != 0) { p->mldp_cq_size_shift_default = MLXCX_CQ_SIZE_SHIFT_DFLT; p->mldp_rq_size_shift_default = MLXCX_RQ_SIZE_SHIFT_DFLT; p->mldp_sq_size_shift_default = MLXCX_SQ_SIZE_SHIFT_DFLT; diff --git a/usr/src/uts/common/io/mlxcx/mlxcx.h b/usr/src/uts/common/io/mlxcx/mlxcx.h index a85a432bbebd..af514cb5b619 100644 --- a/usr/src/uts/common/io/mlxcx/mlxcx.h +++ b/usr/src/uts/common/io/mlxcx/mlxcx.h @@ -13,6 +13,7 @@ * Copyright 2021, The University of Queensland * Copyright (c) 2018, Joyent, Inc. * Copyright 2020 RackTop Systems, Inc. + * Copyright 2023 MNX Cloud, Inc. */ /* @@ -389,6 +390,9 @@ struct mlxcx_port { mlxcx_eth_proto_t mlp_max_proto; mlxcx_eth_proto_t mlp_admin_proto; mlxcx_eth_proto_t mlp_oper_proto; + mlxcx_ext_eth_proto_t mlp_ext_max_proto; + mlxcx_ext_eth_proto_t mlp_ext_admin_proto; + mlxcx_ext_eth_proto_t mlp_ext_oper_proto; mlxcx_pplm_fec_active_t mlp_fec_active; link_fec_t mlp_fec_requested; @@ -1459,7 +1463,8 @@ extern int mlxcx_page_compare(const void *, const void *); extern void mlxcx_update_link_state(mlxcx_t *, mlxcx_port_t *); -extern void mlxcx_eth_proto_to_string(mlxcx_eth_proto_t, char *, size_t); +extern void mlxcx_eth_proto_to_string(mlxcx_eth_proto_t, mlxcx_ext_eth_proto_t, + char *, size_t); extern const char *mlxcx_port_status_string(mlxcx_port_status_t); extern const char *mlxcx_event_name(mlxcx_event_t); diff --git a/usr/src/uts/common/io/mlxcx/mlxcx_cmd.c b/usr/src/uts/common/io/mlxcx/mlxcx_cmd.c index 7ea0de4dabdd..bf0eb4d5d58a 100644 --- a/usr/src/uts/common/io/mlxcx/mlxcx_cmd.c +++ b/usr/src/uts/common/io/mlxcx/mlxcx_cmd.c @@ -13,6 +13,7 @@ * Copyright 2020, The University of Queensland * Copyright (c) 2018, Joyent, Inc. * Copyright 2020 RackTop Systems, Inc. + * Copyright 2023 MNX Cloud, Inc. */ /* @@ -416,7 +417,8 @@ mlxcx_port_status_string(mlxcx_port_status_t st) } void -mlxcx_eth_proto_to_string(mlxcx_eth_proto_t p, char *buf, size_t size) +mlxcx_eth_proto_to_string(mlxcx_eth_proto_t p, mlxcx_ext_eth_proto_t ep, + char *buf, size_t size) { if (p & MLXCX_PROTO_SGMII) (void) strlcat(buf, "SGMII|", size); @@ -446,12 +448,22 @@ mlxcx_eth_proto_to_string(mlxcx_eth_proto_t p, char *buf, size_t size) (void) strlcat(buf, "40GBASE_LR4_ER4|", size); if (p & MLXCX_PROTO_50GBASE_SR2) (void) strlcat(buf, "50GBASE_SR2|", size); + if (p & MLXCX_PROTO_50GBASE_KR4) + (void) strlcat(buf, "50GBASE_KR4|", size); if (p & MLXCX_PROTO_100GBASE_CR4) (void) strlcat(buf, "100GBASE_CR4|", size); if (p & MLXCX_PROTO_100GBASE_SR4) (void) strlcat(buf, "100GBASE_SR4|", size); if (p & MLXCX_PROTO_100GBASE_KR4) (void) strlcat(buf, "100GBASE_KR4|", size); + if (p & MLXCX_PROTO_100GBASE_LR4) + (void) strlcat(buf, "100GBASE_LR4|", size); + if (p & MLXCX_PROTO_100BASE_T) + (void) strlcat(buf, "100BASE_T|", size); + if (p & MLXCX_PROTO_1000BASE_T) + (void) strlcat(buf, "1000BASE_T|", size); + if (p & MLXCX_PROTO_10GBASE_T) + (void) strlcat(buf, "10GBASE_T|", size); if (p & MLXCX_PROTO_25GBASE_CR) (void) strlcat(buf, "25GBASE_CR|", size); if (p & MLXCX_PROTO_25GBASE_KR) @@ -460,6 +472,40 @@ mlxcx_eth_proto_to_string(mlxcx_eth_proto_t p, char *buf, size_t size) (void) strlcat(buf, "25GBASE_SR|", size); if (p & MLXCX_PROTO_50GBASE_CR2) (void) strlcat(buf, "50GBASE_CR2|", size); + + /* Now, for the extended bits... */ + if (ep & MLXCX_EXTPROTO_SGMII_100M) + (void) strlcat(buf, "SGMII_100M|", size); + if (ep & MLXCX_EXTPROTO_1000BASE_X_SGMII) + (void) strlcat(buf, "1000BASE_X_SGMII|", size); + if (ep & MLXCX_EXTPROTO_5GBASE_R) + (void) strlcat(buf, "5GBASE_R|", size); + if (ep & MLXCX_EXTPROTO_10GBASE_XFI_XAUI_1) + (void) strlcat(buf, "10GBASE_XFI_XAUI_1|", size); + if (ep & MLXCX_EXTPROTO_40GBASE_XLAUI_4_XLPPI_4) + (void) strlcat(buf, "40GBASE_XLAUI_4_XLPPI_4|", size); + if (ep & MLXCX_EXTPROTO_25GAUI_1_25GBASE_CR_KR) + (void) strlcat(buf, "25GAUI_1_25GBASE_CR_KR|", size); + if (ep & MLXCX_EXTPROTO_50GAUI_2_LAUI_2_50GBASE_CR2_KR2) + (void) strlcat(buf, "50GAUI_2_LAUI_2_50GBASE_CR2_KR2|", size); + if (ep & MLXCX_EXTPROTO_50GAUI_1_LAUI_1_50GBASE_CR_KR) + (void) strlcat(buf, "50GAUI_1_LAUI_1_50GBASE_CR_KR|", size); + if (ep & MLXCX_EXTPROTO_CAUI_4_100GBASE_CR4_KR4) + (void) strlcat(buf, "CAUI_4_100GBASE_CR4_KR4|", size); + if (ep & MLXCX_EXTPROTO_100GAUI_2_100GBASE_CR2_KR2) + (void) strlcat(buf, "100GAUI_2_100GBASE_CR2_KR2|", size); + if (ep & MLXCX_EXTPROTO_100GAUI_1_100GBASE_CR_KR) + (void) strlcat(buf, "100GAUI_1_100GBASE_CR_KR|", size); + /* Print these if we need 'em for debugging... */ + if (ep & MLXCX_EXTPROTO_200GAUI_4_200GBASE_CR4_KR4) + (void) strlcat(buf, "200GAUI_4_200GBASE_CR4_KR4|", size); + if (ep & MLXCX_EXTPROTO_200GAUI_2_200GBASE_CR2_KR2) + (void) strlcat(buf, "200GAUI_2_200GBASE_CR2_KR2|", size); + if (ep & MLXCX_EXTPROTO_400GAUI_8) + (void) strlcat(buf, "400GAUI_8|", size); + if (ep & MLXCX_EXTPROTO_400GAUI_4_400GBASE_CR4_KR4) + (void) strlcat(buf, "400GAUI_4_400GBASE_CR4_KR4|", size); + /* Chop off the trailing '|' */ if (strlen(buf) > 0) buf[strlen(buf) - 1] = '\0'; @@ -1929,6 +1975,12 @@ mlxcx_cmd_query_port_speed(mlxcx_t *mlxp, mlxcx_port_t *mlp) from_bits32(data.mlrd_ptys.mlrd_ptys_proto_admin); mlp->mlp_oper_proto = from_bits32(data.mlrd_ptys.mlrd_ptys_proto_oper); + mlp->mlp_ext_max_proto = + from_bits32(data.mlrd_ptys.mlrd_ptys_ext_proto_cap); + mlp->mlp_ext_admin_proto = + from_bits32(data.mlrd_ptys.mlrd_ptys_ext_proto_admin); + mlp->mlp_ext_oper_proto = + from_bits32(data.mlrd_ptys.mlrd_ptys_ext_proto_oper); } return (ret); @@ -3717,7 +3769,9 @@ CTASSERT(offsetof(mlxcx_cmd_create_rqt_in_t, mlxi_create_rqt_context) == 0x20); CTASSERT(offsetof(mlxcx_reg_pmtu_t, mlrd_pmtu_oper_mtu) == 0x0C); CTASSERT(sizeof (mlxcx_reg_ptys_t) == 64); +CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_ext_proto_cap) == 0x08); CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_cap) == 0x0c); +CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_ext_proto_admin) == 0x14); CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_admin) == 0x18); CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_partner_advert) == 0x30); diff --git a/usr/src/uts/common/io/mlxcx/mlxcx_gld.c b/usr/src/uts/common/io/mlxcx/mlxcx_gld.c index 26874db48ee1..5490472d50a0 100644 --- a/usr/src/uts/common/io/mlxcx/mlxcx_gld.c +++ b/usr/src/uts/common/io/mlxcx/mlxcx_gld.c @@ -13,6 +13,7 @@ * Copyright (c) 2021, the University of Queensland * Copyright 2020 RackTop Systems, Inc. * Copyright 2023 Oxide Computer Company + * Copyright 2023 MNX Cloud, Inc. */ /* @@ -45,13 +46,27 @@ static char *mlxcx_priv_props[] = { #define GBITS (1000ULL * MBITS) static uint64_t -mlxcx_speed_to_bits(mlxcx_eth_proto_t v) +mlxcx_speed_to_bits(mlxcx_eth_proto_t proto, mlxcx_ext_eth_proto_t ext_proto) { - switch (v) { + /* + * Older parts only used "proto", but starting with ConnectX-6, there + * might be speeds & link-types in an extended set of proto bits. + * + * For now, we check the old ones first, and act accordingly. If we + * get a legitimate single-bit value, we don't worry about ext_proto. + * (That may be something we do need to check, however, in + * appropriately new HW, just in case.) + */ + switch (proto) { + case MLXCX_PROTO_NONE: /* Aka "0" */ + /* Go straight to checking ext_proto. */ + break; case MLXCX_PROTO_SGMII_100BASE: + case MLXCX_PROTO_100BASE_T: return (100ULL * MBITS); case MLXCX_PROTO_SGMII: case MLXCX_PROTO_1000BASE_KX: + case MLXCX_PROTO_1000BASE_T: return (1000ULL * MBITS); case MLXCX_PROTO_10GBASE_CX4: case MLXCX_PROTO_10GBASE_KX4: @@ -59,6 +74,7 @@ mlxcx_speed_to_bits(mlxcx_eth_proto_t v) case MLXCX_PROTO_10GBASE_CR: case MLXCX_PROTO_10GBASE_SR: case MLXCX_PROTO_10GBASE_ER_LR: + case MLXCX_PROTO_10GBASE_T: return (10ULL * GBITS); case MLXCX_PROTO_40GBASE_CR4: case MLXCX_PROTO_40GBASE_KR4: @@ -72,14 +88,66 @@ mlxcx_speed_to_bits(mlxcx_eth_proto_t v) case MLXCX_PROTO_50GBASE_SR2: case MLXCX_PROTO_50GBASE_CR2: case MLXCX_PROTO_50GBASE_KR2: + case MLXCX_PROTO_50GBASE_KR4: return (50ULL * GBITS); case MLXCX_PROTO_100GBASE_CR4: case MLXCX_PROTO_100GBASE_SR4: case MLXCX_PROTO_100GBASE_KR4: + case MLXCX_PROTO_100GBASE_LR4: return (100ULL * GBITS); default: + /* + * We've checked for 0 explicitly above, so don't worry here. + * + * There ARE legitimate single-bit values we don't support, + * and should just return 0 immediately. We will ASSERT() + * that it's a single-bit value, however. + */ + ASSERT0((uint32_t)proto & ((uint32_t)proto - 1U)); return (0); } + + switch (ext_proto) { + case MLXCX_EXTPROTO_SGMII_100M: + return (100ULL * MBITS); + case MLXCX_EXTPROTO_1000BASE_X_SGMII: + return (1000ULL * MBITS); + case MLXCX_EXTPROTO_5GBASE_R: + /* XXX KEBE ASKS - Need more for 5Gbit support here? */ + return (5ULL * GBITS); + case MLXCX_EXTPROTO_10GBASE_XFI_XAUI_1: + return (10ULL * GBITS); + case MLXCX_EXTPROTO_40GBASE_XLAUI_4_XLPPI_4: + return (40ULL * GBITS); + case MLXCX_EXTPROTO_25GAUI_1_25GBASE_CR_KR: + return (25ULL * GBITS); + case MLXCX_EXTPROTO_50GAUI_2_LAUI_2_50GBASE_CR2_KR2: + case MLXCX_EXTPROTO_50GAUI_1_LAUI_1_50GBASE_CR_KR: + return (50ULL * GBITS); + case MLXCX_EXTPROTO_CAUI_4_100GBASE_CR4_KR4: + case MLXCX_EXTPROTO_100GAUI_2_100GBASE_CR2_KR2: + case MLXCX_EXTPROTO_100GAUI_1_100GBASE_CR_KR: + return (100ULL * GBITS); +#if 0 /* Not yet... */ + case MLXCX_EXTPROTO_200GAUI_4_200GBASE_CR4_KR4: + case MLXCX_EXTPROTO_200GAUI_2_200GBASE_CR2_KR2: + case MLXCX_EXTPROTO_400GAUI_8: + case MLXCX_EXTPROTO_400GAUI_4_400GBASE_CR4_KR4: + /* Not yet supported... */ + break; +#endif /* Not yet... */ + default: + /* + * There ARE legitimate single-bit values we don't support, + * and should just return 0 immediately. We will ASSERT() + * that it's a single-bit value, however. + */ + /* This check should work okay for 0 too. */ + ASSERT0((uint32_t)ext_proto & ((uint32_t)ext_proto - 1U)); + break; + } + + return (0); } static link_fec_t @@ -130,9 +198,9 @@ mlxcx_link_fec_cap(link_fec_t fec, mlxcx_pplm_fec_caps_t *pfecp) } static mac_ether_media_t -mlxcx_mac_media(mlxcx_port_status_t st, mlxcx_eth_proto_t v) +mlxcx_mac_media(mlxcx_port_t *port) { - switch (st) { + switch (port->mlp_oper_status) { case MLXCX_PORT_STATUS_UP: case MLXCX_PORT_STATUS_UP_ONCE: break; @@ -142,7 +210,14 @@ mlxcx_mac_media(mlxcx_port_status_t st, mlxcx_eth_proto_t v) return (ETHER_MEDIA_UNKNOWN); } - switch (v) { + /* + * Same issue as mlxcx_speed_to_bits(): this information can be in one + * of two different bit vectors. + */ + switch (port->mlp_oper_proto) { + case MLXCX_PROTO_NONE: /* Aka "0" */ + /* Go straight to checking ext_proto */ + break; case MLXCX_PROTO_SGMII: return (ETHER_MEDIA_1000_SGMII); case MLXCX_PROTO_1000BASE_KX: @@ -177,6 +252,15 @@ mlxcx_mac_media(mlxcx_port_status_t st, mlxcx_eth_proto_t v) return (ETHER_MEDIA_100GBASE_SR4); case MLXCX_PROTO_100GBASE_KR4: return (ETHER_MEDIA_100GBASE_KR4); + case MLXCX_PROTO_100GBASE_LR4: + return (ETHER_MEDIA_100GBASE_LR4); + case MLXCX_PROTO_100BASE_T: + /* See SPEED_100 case in e1000_illumos.c for why we pick TX. */ + return (ETHER_MEDIA_100BASE_TX); + case MLXCX_PROTO_1000BASE_T: + return (ETHER_MEDIA_1000BASE_T); + case MLXCX_PROTO_10GBASE_T: + return (ETHER_MEDIA_10GBASE_T); case MLXCX_PROTO_25GBASE_CR: return (ETHER_MEDIA_25GBASE_CR); case MLXCX_PROTO_25GBASE_KR: @@ -188,8 +272,66 @@ mlxcx_mac_media(mlxcx_port_status_t st, mlxcx_eth_proto_t v) case MLXCX_PROTO_50GBASE_KR2: return (ETHER_MEDIA_50GBASE_KR2); default: + /* + * We've checked for 0 explicitly above, so don't worry here. + * + * There ARE legitimate single-bit values we don't support, + * and should just return 0 immediately. We will ASSERT() + * that it's a single-bit value, however. + */ + ASSERT0((uint32_t)port->mlp_oper_proto & + ((uint32_t)port->mlp_oper_proto - 1U)); + return (ETHER_MEDIA_UNKNOWN); + } + + switch (port->mlp_ext_oper_proto) { + case MLXCX_EXTPROTO_SGMII_100M: + return (ETHER_MEDIA_100_SGMII); + case MLXCX_EXTPROTO_1000BASE_X_SGMII: + return (ETHER_MEDIA_1000_SGMII); + case MLXCX_EXTPROTO_5GBASE_R: + return (ETHER_MEDIA_5000BASE_KR); /* XXX KEBE ASKS use _KR ? */ + case MLXCX_EXTPROTO_10GBASE_XFI_XAUI_1: + return (ETHER_MEDIA_10G_XAUI); + case MLXCX_EXTPROTO_40GBASE_XLAUI_4_XLPPI_4: + return (ETHER_MEDIA_40G_XLPPI); + case MLXCX_EXTPROTO_25GAUI_1_25GBASE_CR_KR: + return (ETHER_MEDIA_25G_AUI); + case MLXCX_EXTPROTO_50GAUI_2_LAUI_2_50GBASE_CR2_KR2: + case MLXCX_EXTPROTO_50GAUI_1_LAUI_1_50GBASE_CR_KR: + /* No type for 50G AUI as far as I can see. */ + return (ETHER_MEDIA_UNKNOWN); + case MLXCX_EXTPROTO_CAUI_4_100GBASE_CR4_KR4: + return (ETHER_MEDIA_100GBASE_CAUI4); + case MLXCX_EXTPROTO_100GAUI_2_100GBASE_CR2_KR2: + case MLXCX_EXTPROTO_100GAUI_1_100GBASE_CR_KR: + /* No type for 100G AUI as far as I can see. */ return (ETHER_MEDIA_UNKNOWN); + /* + * NOTE: These report unsupported but keeping them in active code for + * detection purposes. + */ + case MLXCX_EXTPROTO_200GAUI_4_200GBASE_CR4_KR4: + return (ETHER_MEDIA_200GAUI_4); + case MLXCX_EXTPROTO_200GAUI_2_200GBASE_CR2_KR2: + return (ETHER_MEDIA_200GAUI_2); + case MLXCX_EXTPROTO_400GAUI_8: + return (ETHER_MEDIA_400GAUI_8); + case MLXCX_EXTPROTO_400GAUI_4_400GBASE_CR4_KR4: + return (ETHER_MEDIA_400GAUI_4); + default: + /* + * There ARE legitimate single-bit values we don't support, + * and should just return 0 immediately. We will ASSERT() + * that it's a single-bit value, however. + */ + /* This check should work okay for 0 too. */ + ASSERT0((uint32_t)port->mlp_ext_oper_proto & + ((uint32_t)port->mlp_ext_oper_proto - 1U)); + break; } + + return (ETHER_MEDIA_UNKNOWN); } static int @@ -305,7 +447,8 @@ mlxcx_mac_stat(void *arg, uint_t stat, uint64_t *val) switch (stat) { case MAC_STAT_IFSPEED: - *val = mlxcx_speed_to_bits(port->mlp_oper_proto); + *val = mlxcx_speed_to_bits(port->mlp_oper_proto, + port->mlp_ext_oper_proto); break; case ETHER_STAT_LINK_DUPLEX: *val = LINK_DUPLEX_FULL; @@ -332,8 +475,7 @@ mlxcx_mac_stat(void *arg, uint_t stat, uint64_t *val) *val = port->mlp_stats.mlps_rx_drops; break; case ETHER_STAT_XCVR_INUSE: - *val = (uint64_t)mlxcx_mac_media(port->mlp_oper_status, - port->mlp_oper_proto); + *val = (uint64_t)mlxcx_mac_media(port); break; default: ret = ENOTSUP; @@ -1221,43 +1363,50 @@ mlxcx_mac_propinfo(void *arg, const char *pr_name, mac_prop_id_t pr_num, case MAC_PROP_EN_100GFDX_CAP: mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ); mac_prop_info_set_default_uint8(prh, - (port->mlp_oper_proto & MLXCX_PROTO_100G) != 0); + ((port->mlp_oper_proto & MLXCX_PROTO_100G) != 0 || + (port->mlp_ext_oper_proto & MLXCX_EXTPROTO_100G)) != 0); break; case MAC_PROP_ADV_50GFDX_CAP: case MAC_PROP_EN_50GFDX_CAP: mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ); mac_prop_info_set_default_uint8(prh, - (port->mlp_oper_proto & MLXCX_PROTO_50G) != 0); + ((port->mlp_oper_proto & MLXCX_PROTO_50G) != 0 || + (port->mlp_ext_oper_proto & MLXCX_EXTPROTO_50G)) != 0); break; case MAC_PROP_ADV_40GFDX_CAP: case MAC_PROP_EN_40GFDX_CAP: mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ); mac_prop_info_set_default_uint8(prh, - (port->mlp_oper_proto & MLXCX_PROTO_40G) != 0); + ((port->mlp_oper_proto & MLXCX_PROTO_40G) != 0 || + (port->mlp_ext_oper_proto & MLXCX_EXTPROTO_40G)) != 0); break; case MAC_PROP_ADV_25GFDX_CAP: case MAC_PROP_EN_25GFDX_CAP: mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ); mac_prop_info_set_default_uint8(prh, - (port->mlp_oper_proto & MLXCX_PROTO_25G) != 0); + ((port->mlp_oper_proto & MLXCX_PROTO_25G) != 0 || + (port->mlp_ext_oper_proto & MLXCX_EXTPROTO_25G)) != 0); break; case MAC_PROP_ADV_10GFDX_CAP: case MAC_PROP_EN_10GFDX_CAP: mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ); mac_prop_info_set_default_uint8(prh, - (port->mlp_oper_proto & MLXCX_PROTO_10G) != 0); + ((port->mlp_oper_proto & MLXCX_PROTO_10G) != 0 || + (port->mlp_ext_oper_proto & MLXCX_EXTPROTO_10G)) != 0); break; case MAC_PROP_ADV_1000FDX_CAP: case MAC_PROP_EN_1000FDX_CAP: mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ); mac_prop_info_set_default_uint8(prh, - (port->mlp_oper_proto & MLXCX_PROTO_1G) != 0); + ((port->mlp_oper_proto & MLXCX_PROTO_1G) != 0 || + (port->mlp_ext_oper_proto & MLXCX_EXTPROTO_1G)) != 0); break; case MAC_PROP_ADV_100FDX_CAP: case MAC_PROP_EN_100FDX_CAP: mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ); mac_prop_info_set_default_uint8(prh, - (port->mlp_oper_proto & MLXCX_PROTO_100M) != 0); + ((port->mlp_oper_proto & MLXCX_PROTO_100M) != 0 || + (port->mlp_ext_oper_proto & MLXCX_EXTPROTO_100M)) != 0); break; default: break; @@ -1408,7 +1557,8 @@ mlxcx_mac_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, ret = EOVERFLOW; break; } - speed = mlxcx_speed_to_bits(port->mlp_oper_proto); + speed = mlxcx_speed_to_bits(port->mlp_oper_proto, + port->mlp_ext_oper_proto); bcopy(&speed, pr_val, sizeof (speed)); break; case MAC_PROP_STATUS: @@ -1429,9 +1579,7 @@ mlxcx_mac_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, } break; case MAC_PROP_MEDIA: - *(mac_ether_media_t *)pr_val = - mlxcx_mac_media(port->mlp_oper_status, - port->mlp_oper_proto); + *(mac_ether_media_t *)pr_val = mlxcx_mac_media(port); break; case MAC_PROP_AUTONEG: if (pr_valsize < sizeof (uint8_t)) { @@ -1469,7 +1617,8 @@ mlxcx_mac_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, break; } *(uint8_t *)pr_val = (port->mlp_max_proto & - MLXCX_PROTO_100G) != 0; + MLXCX_PROTO_100G) != 0 || + (port->mlp_ext_max_proto & MLXCX_EXTPROTO_100G) != 0; break; case MAC_PROP_ADV_50GFDX_CAP: case MAC_PROP_EN_50GFDX_CAP: @@ -1478,7 +1627,8 @@ mlxcx_mac_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, break; } *(uint8_t *)pr_val = (port->mlp_max_proto & - MLXCX_PROTO_50G) != 0; + MLXCX_PROTO_50G) != 0 || + (port->mlp_ext_max_proto & MLXCX_EXTPROTO_50G) != 0; break; case MAC_PROP_ADV_40GFDX_CAP: case MAC_PROP_EN_40GFDX_CAP: @@ -1487,7 +1637,8 @@ mlxcx_mac_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, break; } *(uint8_t *)pr_val = (port->mlp_max_proto & - MLXCX_PROTO_40G) != 0; + MLXCX_PROTO_40G) != 0 || + (port->mlp_ext_max_proto & MLXCX_EXTPROTO_40G) != 0; break; case MAC_PROP_ADV_25GFDX_CAP: case MAC_PROP_EN_25GFDX_CAP: @@ -1496,7 +1647,8 @@ mlxcx_mac_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, break; } *(uint8_t *)pr_val = (port->mlp_max_proto & - MLXCX_PROTO_25G) != 0; + MLXCX_PROTO_25G) != 0 || + (port->mlp_ext_max_proto & MLXCX_EXTPROTO_25G) != 0; break; case MAC_PROP_ADV_10GFDX_CAP: case MAC_PROP_EN_10GFDX_CAP: @@ -1505,7 +1657,8 @@ mlxcx_mac_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, break; } *(uint8_t *)pr_val = (port->mlp_max_proto & - MLXCX_PROTO_10G) != 0; + MLXCX_PROTO_10G) != 0 || + (port->mlp_ext_max_proto & MLXCX_EXTPROTO_10G) != 0; break; case MAC_PROP_ADV_1000FDX_CAP: case MAC_PROP_EN_1000FDX_CAP: @@ -1514,7 +1667,8 @@ mlxcx_mac_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, break; } *(uint8_t *)pr_val = (port->mlp_max_proto & - MLXCX_PROTO_1G) != 0; + MLXCX_PROTO_1G) != 0 || + (port->mlp_ext_max_proto & MLXCX_EXTPROTO_1G) != 0; break; case MAC_PROP_ADV_100FDX_CAP: case MAC_PROP_EN_100FDX_CAP: @@ -1523,7 +1677,8 @@ mlxcx_mac_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, break; } *(uint8_t *)pr_val = (port->mlp_max_proto & - MLXCX_PROTO_100M) != 0; + MLXCX_PROTO_100M) != 0 || + (port->mlp_ext_max_proto & MLXCX_EXTPROTO_100M) != 0; break; default: ret = ENOTSUP; diff --git a/usr/src/uts/common/io/mlxcx/mlxcx_reg.h b/usr/src/uts/common/io/mlxcx/mlxcx_reg.h index 24435a22ab1d..4bca99246d9b 100644 --- a/usr/src/uts/common/io/mlxcx/mlxcx_reg.h +++ b/usr/src/uts/common/io/mlxcx/mlxcx_reg.h @@ -13,6 +13,7 @@ * Copyright 2020, The University of Queensland * Copyright (c) 2018, Joyent, Inc. * Copyright 2020 RackTop Systems, Inc. + * Copyright 2023 MNX Cloud, Inc. */ #ifndef _MLXCX_REG_H @@ -1372,16 +1373,20 @@ typedef struct { uint8_t mlcap_general_rsvd6[1]; uint8_t mlcap_general_log_max_cq_sz; - uint8_t mlcap_general_rsvd7[1]; + /* XXX NOTE: relaxed_ordering_umr is not relaxed_ordering, per se. */ + uint8_t mlcap_general_relaxed_ordering_umr; uint8_t mlcap_general_log_max_cq; - + uint8_t mlcap_general_log_max_eq_sz; + /* XXX NOTE: bits 0xc0 of mkey_flags are now for relaxed_ordering. */ uint8_t mlcap_general_log_max_mkey_flags; uint8_t mlcap_general_rsvd8[1]; + /* XXX NOTE: bit 0x80 of max_eq is now for fast_teardown. */ uint8_t mlcap_general_log_max_eq; uint8_t mlcap_general_max_indirection; uint8_t mlcap_general_log_max_mrw_sz_flags; + /* XXX NOTE: bit 0x80 of bsf_list is now for force_teardown. */ uint8_t mlcap_general_log_max_bsf_list_size_flags; uint8_t mlcap_general_log_max_klm_list_size_flags; @@ -1398,6 +1403,7 @@ typedef struct { uint16be_t mlcap_general_flags_a; uint16be_t mlcap_general_gid_table_size; + /* XXX NOTE: bits 0x3ff of flags_b are for max_qp_count */ bits16_t mlcap_general_flags_b; uint16be_t mlcap_general_pkey_table_size; @@ -2251,16 +2257,18 @@ typedef struct { uint8_t mlrd_paos_rsvd[8]; } mlxcx_reg_paos_t; +/* Some values are derived from the FreeBSD `mlx5` directory. */ typedef enum { + MLXCX_PROTO_NONE = 0, MLXCX_PROTO_SGMII = 1 << 0, MLXCX_PROTO_1000BASE_KX = 1 << 1, MLXCX_PROTO_10GBASE_CX4 = 1 << 2, MLXCX_PROTO_10GBASE_KX4 = 1 << 3, MLXCX_PROTO_10GBASE_KR = 1 << 4, - MLXCX_PROTO_UNKNOWN_1 = 1 << 5, + MLXCX_PROTO_20GBASE_KR2 = 1 << 5, /* Unsupported */ MLXCX_PROTO_40GBASE_CR4 = 1 << 6, MLXCX_PROTO_40GBASE_KR4 = 1 << 7, - MLXCX_PROTO_UNKNOWN_2 = 1 << 8, + MLXCX_PROTO_56GBASE_R4 = 1 << 8, /* Unsupported */ MLXCX_PROTO_SGMII_100BASE = 1 << 9, MLXCX_PROTO_UNKNOWN_3 = 1 << 10, MLXCX_PROTO_UNKNOWN_4 = 1 << 11, @@ -2271,14 +2279,14 @@ typedef enum { MLXCX_PROTO_40GBASE_LR4_ER4 = 1 << 16, MLXCX_PROTO_UNKNOWN_5 = 1 << 17, MLXCX_PROTO_50GBASE_SR2 = 1 << 18, - MLXCX_PROTO_UNKNOWN_6 = 1 << 19, + MLXCX_PROTO_50GBASE_KR4 = 1 << 19, MLXCX_PROTO_100GBASE_CR4 = 1 << 20, MLXCX_PROTO_100GBASE_SR4 = 1 << 21, MLXCX_PROTO_100GBASE_KR4 = 1 << 22, - MLXCX_PROTO_UNKNOWN_7 = 1 << 23, - MLXCX_PROTO_UNKNOWN_8 = 1 << 24, - MLXCX_PROTO_UNKNOWN_9 = 1 << 25, - MLXCX_PROTO_UNKNOWN_10 = 1 << 26, + MLXCX_PROTO_100GBASE_LR4 = 1 << 23, + MLXCX_PROTO_100BASE_T = 1 << 24, + MLXCX_PROTO_1000BASE_T = 1 << 25, + MLXCX_PROTO_10GBASE_T = 1 << 26, MLXCX_PROTO_25GBASE_CR = 1 << 27, MLXCX_PROTO_25GBASE_KR = 1 << 28, MLXCX_PROTO_25GBASE_SR = 1 << 29, @@ -2286,27 +2294,64 @@ typedef enum { MLXCX_PROTO_50GBASE_KR2 = 1UL << 31, } mlxcx_eth_proto_t; -#define MLXCX_PROTO_100M MLXCX_PROTO_SGMII_100BASE +/* Values are derived from the FreeBSD `mlx5` directory. */ +typedef enum { + MLXCX_EXTPROTO_NONE = 0, + MLXCX_EXTPROTO_SGMII_100M = 1UL << 0, + MLXCX_EXTPROTO_1000BASE_X_SGMII = 1UL << 1, + /* 1UL << 2 */ + MLXCX_EXTPROTO_5GBASE_R = 1UL << 3, + MLXCX_EXTPROTO_10GBASE_XFI_XAUI_1 = 1UL << 4, + MLXCX_EXTPROTO_40GBASE_XLAUI_4_XLPPI_4 = 1UL << 5, + MLXCX_EXTPROTO_25GAUI_1_25GBASE_CR_KR = 1UL << 6, + MLXCX_EXTPROTO_50GAUI_2_LAUI_2_50GBASE_CR2_KR2 = 1UL << 7, + MLXCX_EXTPROTO_50GAUI_1_LAUI_1_50GBASE_CR_KR = 1UL << 8, + MLXCX_EXTPROTO_CAUI_4_100GBASE_CR4_KR4 = 1UL << 9, + MLXCX_EXTPROTO_100GAUI_2_100GBASE_CR2_KR2 = 1UL << 10, + MLXCX_EXTPROTO_100GAUI_1_100GBASE_CR_KR = 1UL << 11, + MLXCX_EXTPROTO_200GAUI_4_200GBASE_CR4_KR4 = 1UL << 12, + MLXCX_EXTPROTO_200GAUI_2_200GBASE_CR2_KR2 = 1UL << 13, + /* 1UL << 14 */ + MLXCX_EXTPROTO_400GAUI_8 = 1UL << 15, + MLXCX_EXTPROTO_400GAUI_4_400GBASE_CR4_KR4 = 1UL << 16, + /* 17-31 */ +} mlxcx_ext_eth_proto_t; + +#define MLXCX_PROTO_100M (MLXCX_PROTO_SGMII_100BASE | \ + MLXCX_PROTO_100BASE_T) +#define MLXCX_EXTPROTO_100M MLXCX_EXTPROTO_SGMII_100M #define MLXCX_PROTO_1G (MLXCX_PROTO_1000BASE_KX | MLXCX_PROTO_SGMII) +#define MLXCX_EXTPROTO_1G MLXCX_EXTPROTO_1000BASE_X_SGMII + +#define MLXCX_EXTPROTO_5G MLXCX_EXTPROTO_5GBASE_R #define MLXCX_PROTO_10G (MLXCX_PROTO_10GBASE_CX4 | \ MLXCX_PROTO_10GBASE_KX4 | MLXCX_PROTO_10GBASE_KR | \ MLXCX_PROTO_10GBASE_CR | MLXCX_PROTO_10GBASE_SR | \ MLXCX_PROTO_10GBASE_ER_LR) +#define MLXCX_EXTPROTO_10G MLXCX_EXTPROTO_10GBASE_XFI_XAUI_1 #define MLXCX_PROTO_25G (MLXCX_PROTO_25GBASE_CR | \ MLXCX_PROTO_25GBASE_KR | MLXCX_PROTO_25GBASE_SR) +#define MLXCX_EXTPROTO_25G MLXCX_EXTPROTO_25GAUI_1_25GBASE_CR_KR #define MLXCX_PROTO_40G (MLXCX_PROTO_40GBASE_SR4 | \ MLXCX_PROTO_40GBASE_LR4_ER4 | MLXCX_PROTO_40GBASE_CR4 | \ MLXCX_PROTO_40GBASE_KR4) +#define MLXCX_EXTPROTO_40G MLXCX_EXTPROTO_40GBASE_XLAUI_4_XLPPI_4 #define MLXCX_PROTO_50G (MLXCX_PROTO_50GBASE_CR2 | \ MLXCX_PROTO_50GBASE_KR2 | MLXCX_PROTO_50GBASE_SR2) +#define MLXCX_EXTPROTO_50G (MLXCX_EXTPROTO_50GAUI_2_LAUI_2_50GBASE_CR2_KR2 | \ + MLXCX_EXTPROTO_50GAUI_1_LAUI_1_50GBASE_CR_KR) #define MLXCX_PROTO_100G (MLXCX_PROTO_100GBASE_CR4 | \ MLXCX_PROTO_100GBASE_SR4 | MLXCX_PROTO_100GBASE_KR4) +#define MLXCX_EXTPROTO_100G ( MLXCX_EXTPROTO_CAUI_4_100GBASE_CR4_KR4 | \ + MLXCX_EXTPROTO_100GAUI_2_100GBASE_CR2_KR2 | \ + MLXCX_EXTPROTO_100GAUI_1_100GBASE_CR_KR) + typedef enum { MLXCX_AUTONEG_DISABLE_CAP = 1 << 5, @@ -2328,12 +2373,13 @@ typedef struct { uint8_t mlrd_ptys_rsvd2; uint16be_t mlrd_ptys_data_rate_oper; - uint8_t mlrd_ptys_rsvd3[4]; - + bits32_t mlrd_ptys_ext_proto_cap; bits32_t mlrd_ptys_proto_cap; - uint8_t mlrd_ptys_rsvd4[8]; + uint8_t mlrd_ptys_rsvd4[4]; + bits32_t mlrd_ptys_ext_proto_admin; bits32_t mlrd_ptys_proto_admin; - uint8_t mlrd_ptys_rsvd5[8]; + uint8_t mlrd_ptys_rsvd5[4]; + bits32_t mlrd_ptys_ext_proto_oper; bits32_t mlrd_ptys_proto_oper; uint8_t mlrd_ptys_rsvd6[8]; bits32_t mlrd_ptys_proto_partner_advert; @@ -2568,6 +2614,7 @@ typedef enum { MLXCX_REG_MCIA = 0x9014, MLXCX_REG_PPCNT = 0x5008, MLXCX_REG_PPLM = 0x5023, + MLXCX_REG_PCAM = 0x507f, MLXCX_REG_MTCAP = 0x9009, MLXCX_REG_MTMP = 0x900A } mlxcx_register_id_t;