Merge branch 'mlxsw-add-port-range-matching-support'
authorJakub Kicinski <kuba@kernel.org>
Wed, 12 Jul 2023 23:57:23 +0000 (16:57 -0700)
committerJakub Kicinski <kuba@kernel.org>
Wed, 12 Jul 2023 23:57:24 +0000 (16:57 -0700)
Petr Machata says:

====================
mlxsw: Add port range matching support

Ido Schimmel writes:

Add port range matching support in mlxsw as part of tc-flower offload.

Patches #1-#7 gradually add port range matching support in mlxsw. See
patch #3 to understand how port range matching is implemented in the
device.

Patches #8-#10 add selftests.
====================

Link: https://lore.kernel.org/r/cover.1689092769.git.petrm@nvidia.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
20 files changed:
drivers/net/ethernet/mellanox/mlxsw/Makefile
drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c
drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h
drivers/net/ethernet/mellanox/mlxsw/reg.h
drivers/net/ethernet/mellanox/mlxsw/resources.h
drivers/net/ethernet/mellanox/mlxsw/spectrum.c
drivers/net/ethernet/mellanox/mlxsw/spectrum.h
drivers/net/ethernet/mellanox/mlxsw/spectrum1_acl_tcam.c
drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c
drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c
drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c
drivers/net/ethernet/mellanox/mlxsw/spectrum_port_range.c [new file with mode: 0644]
tools/testing/selftests/drivers/net/mlxsw/port_range_occ.sh [new file with mode: 0755]
tools/testing/selftests/drivers/net/mlxsw/port_range_scale.sh [new file with mode: 0644]
tools/testing/selftests/drivers/net/mlxsw/spectrum-2/port_range_scale.sh [new symlink]
tools/testing/selftests/drivers/net/mlxsw/spectrum-2/resource_scale.sh
tools/testing/selftests/drivers/net/mlxsw/spectrum/port_range_scale.sh [new file with mode: 0644]
tools/testing/selftests/drivers/net/mlxsw/spectrum/resource_scale.sh
tools/testing/selftests/net/forwarding/Makefile
tools/testing/selftests/net/forwarding/tc_flower_port_range.sh [new file with mode: 0755]

index 3ca9fce..71cad6b 100644 (file)
@@ -29,7 +29,7 @@ mlxsw_spectrum-objs           := spectrum.o spectrum_buffers.o \
                                   spectrum_nve.o spectrum_nve_vxlan.o \
                                   spectrum_dpipe.o spectrum_trap.o \
                                   spectrum_ethtool.o spectrum_policer.o \
-                                  spectrum_pgt.o
+                                  spectrum_pgt.o spectrum_port_range.o
 mlxsw_spectrum-$(CONFIG_MLXSW_SPECTRUM_DCB)    += spectrum_dcb.o
 mlxsw_spectrum-$(CONFIG_PTP_1588_CLOCK)                += spectrum_ptp.o
 obj-$(CONFIG_MLXSW_MINIMAL)    += mlxsw_minimal.o
index f0b2963..7870327 100644 (file)
@@ -43,6 +43,7 @@ static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = {
        MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_32_63, 0x38, 4),
        MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_0_31, 0x3C, 4),
        MLXSW_AFK_ELEMENT_INFO_U32(FDB_MISS, 0x40, 0, 1),
+       MLXSW_AFK_ELEMENT_INFO_U32(L4_PORT_RANGE, 0x40, 1, 16),
 };
 
 struct mlxsw_afk {
index 65a4aba..2eac758 100644 (file)
@@ -36,6 +36,7 @@ enum mlxsw_afk_element {
        MLXSW_AFK_ELEMENT_VIRT_ROUTER_MSB,
        MLXSW_AFK_ELEMENT_VIRT_ROUTER_LSB,
        MLXSW_AFK_ELEMENT_FDB_MISS,
+       MLXSW_AFK_ELEMENT_L4_PORT_RANGE,
        MLXSW_AFK_ELEMENT_MAX,
 };
 
index 8165bf3..df63f39 100644 (file)
@@ -2799,6 +2799,78 @@ static inline void mlxsw_reg_ptar_unpack(char *payload, char *tcam_region_info)
        mlxsw_reg_ptar_tcam_region_info_memcpy_from(payload, tcam_region_info);
 }
 
+/* PPRR - Policy-Engine Port Range Register
+ * ----------------------------------------
+ * This register is used for configuring port range identification.
+ */
+#define MLXSW_REG_PPRR_ID 0x3008
+#define MLXSW_REG_PPRR_LEN 0x14
+
+MLXSW_REG_DEFINE(pprr, MLXSW_REG_PPRR_ID, MLXSW_REG_PPRR_LEN);
+
+/* reg_pprr_ipv4
+ * Apply port range register to IPv4 packets.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, pprr, ipv4, 0x00, 31, 1);
+
+/* reg_pprr_ipv6
+ * Apply port range register to IPv6 packets.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, pprr, ipv6, 0x00, 30, 1);
+
+/* reg_pprr_src
+ * Apply port range register to source L4 ports.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, pprr, src, 0x00, 29, 1);
+
+/* reg_pprr_dst
+ * Apply port range register to destination L4 ports.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, pprr, dst, 0x00, 28, 1);
+
+/* reg_pprr_tcp
+ * Apply port range register to TCP packets.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, pprr, tcp, 0x00, 27, 1);
+
+/* reg_pprr_udp
+ * Apply port range register to UDP packets.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, pprr, udp, 0x00, 26, 1);
+
+/* reg_pprr_register_index
+ * Index of Port Range Register being accessed.
+ * Range is 0..cap_max_acl_l4_port_range-1.
+ * Access: Index
+ */
+MLXSW_ITEM32(reg, pprr, register_index, 0x00, 0, 8);
+
+/* reg_prrr_port_range_min
+ * Minimum port range for comparison.
+ * Match is defined as:
+ * port_range_min <= packet_port <= port_range_max.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, pprr, port_range_min, 0x04, 16, 16);
+
+/* reg_prrr_port_range_max
+ * Maximum port range for comparison.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, pprr, port_range_max, 0x04, 0, 16);
+
+static inline void mlxsw_reg_pprr_pack(char *payload, u8 register_index)
+{
+       MLXSW_REG_ZERO(pprr, payload);
+       mlxsw_reg_pprr_register_index_set(payload, register_index);
+}
+
 /* PPBS - Policy-Engine Policy Based Switching Register
  * ----------------------------------------------------
  * This register retrieves and sets Policy Based Switching Table entries.
@@ -12819,6 +12891,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = {
        MLXSW_REG(pacl),
        MLXSW_REG(pagt),
        MLXSW_REG(ptar),
+       MLXSW_REG(pprr),
        MLXSW_REG(ppbs),
        MLXSW_REG(prcr),
        MLXSW_REG(pefa),
index 19ae0d1..89dd277 100644 (file)
@@ -39,6 +39,7 @@ enum mlxsw_res_id {
        MLXSW_RES_ID_ACL_FLEX_KEYS,
        MLXSW_RES_ID_ACL_MAX_ACTION_PER_RULE,
        MLXSW_RES_ID_ACL_ACTIONS_PER_SET,
+       MLXSW_RES_ID_ACL_MAX_L4_PORT_RANGE,
        MLXSW_RES_ID_ACL_MAX_ERPT_BANKS,
        MLXSW_RES_ID_ACL_MAX_ERPT_BANK_SIZE,
        MLXSW_RES_ID_ACL_MAX_LARGE_KEY_ID,
@@ -99,6 +100,7 @@ static u16 mlxsw_res_ids[] = {
        [MLXSW_RES_ID_ACL_FLEX_KEYS] = 0x2910,
        [MLXSW_RES_ID_ACL_MAX_ACTION_PER_RULE] = 0x2911,
        [MLXSW_RES_ID_ACL_ACTIONS_PER_SET] = 0x2912,
+       [MLXSW_RES_ID_ACL_MAX_L4_PORT_RANGE] = 0x2920,
        [MLXSW_RES_ID_ACL_MAX_ERPT_BANKS] = 0x2940,
        [MLXSW_RES_ID_ACL_MAX_ERPT_BANK_SIZE] = 0x2941,
        [MLXSW_RES_ID_ACL_MAX_LARGE_KEY_ID] = 0x2942,
index 25a01da..86e2f0e 100644 (file)
@@ -3188,6 +3188,12 @@ static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core,
                goto err_nve_init;
        }
 
+       err = mlxsw_sp_port_range_init(mlxsw_sp);
+       if (err) {
+               dev_err(mlxsw_sp->bus_info->dev, "Failed to initialize port ranges\n");
+               goto err_port_range_init;
+       }
+
        err = mlxsw_sp_acl_init(mlxsw_sp);
        if (err) {
                dev_err(mlxsw_sp->bus_info->dev, "Failed to initialize ACL\n");
@@ -3280,6 +3286,8 @@ err_ptp_clock_init:
 err_router_init:
        mlxsw_sp_acl_fini(mlxsw_sp);
 err_acl_init:
+       mlxsw_sp_port_range_fini(mlxsw_sp);
+err_port_range_init:
        mlxsw_sp_nve_fini(mlxsw_sp);
 err_nve_init:
        mlxsw_sp_ipv6_addr_ht_fini(mlxsw_sp);
@@ -3462,6 +3470,7 @@ static void mlxsw_sp_fini(struct mlxsw_core *mlxsw_core)
        }
        mlxsw_sp_router_fini(mlxsw_sp);
        mlxsw_sp_acl_fini(mlxsw_sp);
+       mlxsw_sp_port_range_fini(mlxsw_sp);
        mlxsw_sp_nve_fini(mlxsw_sp);
        mlxsw_sp_ipv6_addr_ht_fini(mlxsw_sp);
        mlxsw_sp_afa_fini(mlxsw_sp);
@@ -3730,6 +3739,26 @@ static int mlxsw_sp_resources_rifs_register(struct mlxsw_core *mlxsw_core)
                                      &size_params);
 }
 
+static int
+mlxsw_sp_resources_port_range_register(struct mlxsw_core *mlxsw_core)
+{
+       struct devlink *devlink = priv_to_devlink(mlxsw_core);
+       struct devlink_resource_size_params size_params;
+       u64 max;
+
+       if (!MLXSW_CORE_RES_VALID(mlxsw_core, ACL_MAX_L4_PORT_RANGE))
+               return -EIO;
+
+       max = MLXSW_CORE_RES_GET(mlxsw_core, ACL_MAX_L4_PORT_RANGE);
+       devlink_resource_size_params_init(&size_params, max, max, 1,
+                                         DEVLINK_RESOURCE_UNIT_ENTRY);
+
+       return devl_resource_register(devlink, "port_range_registers", max,
+                                     MLXSW_SP_RESOURCE_PORT_RANGE_REGISTERS,
+                                     DEVLINK_RESOURCE_ID_PARENT_TOP,
+                                     &size_params);
+}
+
 static int mlxsw_sp1_resources_register(struct mlxsw_core *mlxsw_core)
 {
        int err;
@@ -3758,8 +3787,13 @@ static int mlxsw_sp1_resources_register(struct mlxsw_core *mlxsw_core)
        if (err)
                goto err_resources_rifs_register;
 
+       err = mlxsw_sp_resources_port_range_register(mlxsw_core);
+       if (err)
+               goto err_resources_port_range_register;
+
        return 0;
 
+err_resources_port_range_register:
 err_resources_rifs_register:
 err_resources_rif_mac_profile_register:
 err_policer_resources_register:
@@ -3797,8 +3831,13 @@ static int mlxsw_sp2_resources_register(struct mlxsw_core *mlxsw_core)
        if (err)
                goto err_resources_rifs_register;
 
+       err = mlxsw_sp_resources_port_range_register(mlxsw_core);
+       if (err)
+               goto err_resources_port_range_register;
+
        return 0;
 
+err_resources_port_range_register:
 err_resources_rifs_register:
 err_resources_rif_mac_profile_register:
 err_policer_resources_register:
index 231e364..c6231e6 100644 (file)
@@ -69,6 +69,7 @@ enum mlxsw_sp_resource_id {
        MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS,
        MLXSW_SP_RESOURCE_RIF_MAC_PROFILES,
        MLXSW_SP_RESOURCE_RIFS,
+       MLXSW_SP_RESOURCE_PORT_RANGE_REGISTERS,
 };
 
 struct mlxsw_sp_port;
@@ -175,6 +176,7 @@ struct mlxsw_sp {
        struct mlxsw_sp_acl *acl;
        struct mlxsw_sp_fid_core *fid_core;
        struct mlxsw_sp_policer_core *policer_core;
+       struct mlxsw_sp_port_range_core *pr_core;
        struct mlxsw_sp_kvdl *kvdl;
        struct mlxsw_sp_nve *nve;
        struct notifier_block netdevice_nb;
@@ -865,9 +867,13 @@ struct mlxsw_sp_acl_rule_info {
           egress_bind_blocker:1,
           counter_valid:1,
           policer_index_valid:1,
-          ipv6_valid:1;
+          ipv6_valid:1,
+          src_port_range_reg_valid:1,
+          dst_port_range_reg_valid:1;
        unsigned int counter_index;
        u16 policer_index;
+       u8 src_port_range_reg_index;
+       u8 dst_port_range_reg_index;
        struct {
                u32 prev_val;
                enum mlxsw_sp_acl_mangle_field prev_field;
@@ -992,7 +998,8 @@ void mlxsw_sp_acl_ruleset_prio_get(struct mlxsw_sp_acl_ruleset *ruleset,
 struct mlxsw_sp_acl_rule_info *
 mlxsw_sp_acl_rulei_create(struct mlxsw_sp_acl *acl,
                          struct mlxsw_afa_block *afa_block);
-void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp_acl_rule_info *rulei);
+void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp *mlxsw_sp,
+                               struct mlxsw_sp_acl_rule_info *rulei);
 int mlxsw_sp_acl_rulei_commit(struct mlxsw_sp_acl_rule_info *rulei);
 void mlxsw_sp_acl_rulei_priority(struct mlxsw_sp_acl_rule_info *rulei,
                                 unsigned int priority);
@@ -1484,4 +1491,18 @@ int mlxsw_sp_pgt_entry_port_set(struct mlxsw_sp *mlxsw_sp, u16 mid,
 int mlxsw_sp_pgt_init(struct mlxsw_sp *mlxsw_sp);
 void mlxsw_sp_pgt_fini(struct mlxsw_sp *mlxsw_sp);
 
+/* spectrum_port_range.c */
+struct mlxsw_sp_port_range {
+       u16 min;
+       u16 max;
+       u8 source:1;    /* Source or destination */
+};
+
+int mlxsw_sp_port_range_reg_get(struct mlxsw_sp *mlxsw_sp,
+                               const struct mlxsw_sp_port_range *range,
+                               struct netlink_ext_ack *extack,
+                               u8 *p_prr_index);
+void mlxsw_sp_port_range_reg_put(struct mlxsw_sp *mlxsw_sp, u8 prr_index);
+int mlxsw_sp_port_range_init(struct mlxsw_sp *mlxsw_sp);
+void mlxsw_sp_port_range_fini(struct mlxsw_sp *mlxsw_sp);
 #endif
index 3a636f7..dfcdd37 100644 (file)
@@ -90,7 +90,7 @@ mlxsw_sp1_acl_ctcam_region_catchall_add(struct mlxsw_sp *mlxsw_sp,
 err_entry_add:
 err_rulei_commit:
 err_rulei_act_continue:
-       mlxsw_sp_acl_rulei_destroy(rulei);
+       mlxsw_sp_acl_rulei_destroy(mlxsw_sp, rulei);
 err_rulei_create:
        mlxsw_sp_acl_ctcam_chunk_fini(&region->catchall.cchunk);
        return err;
@@ -105,7 +105,7 @@ mlxsw_sp1_acl_ctcam_region_catchall_del(struct mlxsw_sp *mlxsw_sp,
        mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, &region->cregion,
                                     &region->catchall.cchunk,
                                     &region->catchall.centry);
-       mlxsw_sp_acl_rulei_destroy(rulei);
+       mlxsw_sp_acl_rulei_destroy(mlxsw_sp, rulei);
        mlxsw_sp_acl_ctcam_chunk_fini(&region->catchall.cchunk);
 }
 
index 0423ac2..186161a 100644 (file)
@@ -339,10 +339,17 @@ err_afa_block_create:
        return ERR_PTR(err);
 }
 
-void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp_acl_rule_info *rulei)
+void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp *mlxsw_sp,
+                               struct mlxsw_sp_acl_rule_info *rulei)
 {
        if (rulei->action_created)
                mlxsw_afa_block_destroy(rulei->act_block);
+       if (rulei->src_port_range_reg_valid)
+               mlxsw_sp_port_range_reg_put(mlxsw_sp,
+                                           rulei->src_port_range_reg_index);
+       if (rulei->dst_port_range_reg_valid)
+               mlxsw_sp_port_range_reg_put(mlxsw_sp,
+                                           rulei->dst_port_range_reg_index);
        kfree(rulei);
 }
 
@@ -834,7 +841,7 @@ void mlxsw_sp_acl_rule_destroy(struct mlxsw_sp *mlxsw_sp,
 {
        struct mlxsw_sp_acl_ruleset *ruleset = rule->ruleset;
 
-       mlxsw_sp_acl_rulei_destroy(rule->rulei);
+       mlxsw_sp_acl_rulei_destroy(mlxsw_sp, rule->rulei);
        kfree(rule);
        mlxsw_sp_acl_ruleset_ref_dec(mlxsw_sp, ruleset);
 }
index 4dea39f..b7f5860 100644 (file)
@@ -31,12 +31,14 @@ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_l2_smac_ex[] = {
 
 static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_sip[] = {
        MLXSW_AFK_ELEMENT_INST_BUF(SRC_IP_0_31, 0x00, 4),
+       MLXSW_AFK_ELEMENT_INST_U32(L4_PORT_RANGE, 0x04, 16, 16),
        MLXSW_AFK_ELEMENT_INST_U32(IP_PROTO, 0x08, 0, 8),
        MLXSW_AFK_ELEMENT_INST_U32(SRC_SYS_PORT, 0x0C, 0, 16),
 };
 
 static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_dip[] = {
        MLXSW_AFK_ELEMENT_INST_BUF(DST_IP_0_31, 0x00, 4),
+       MLXSW_AFK_ELEMENT_INST_U32(L4_PORT_RANGE, 0x04, 16, 16),
        MLXSW_AFK_ELEMENT_INST_U32(IP_PROTO, 0x08, 0, 8),
        MLXSW_AFK_ELEMENT_INST_U32(SRC_SYS_PORT, 0x0C, 0, 16),
 };
@@ -205,6 +207,7 @@ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_l4_0[] = {
 
 static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_l4_2[] = {
        MLXSW_AFK_ELEMENT_INST_U32(TCP_FLAGS, 0x04, 16, 9), /* TCP_CONTROL + TCP_ECN */
+       MLXSW_AFK_ELEMENT_INST_U32(L4_PORT_RANGE, 0x04, 0, 16),
 };
 
 static const struct mlxsw_afk_block mlxsw_sp2_afk_blocks[] = {
index 72917f0..8329100 100644 (file)
@@ -418,6 +418,68 @@ static int mlxsw_sp_flower_parse_ports(struct mlxsw_sp *mlxsw_sp,
        return 0;
 }
 
+static int
+mlxsw_sp_flower_parse_ports_range(struct mlxsw_sp *mlxsw_sp,
+                                 struct mlxsw_sp_acl_rule_info *rulei,
+                                 struct flow_cls_offload *f, u8 ip_proto)
+{
+       const struct flow_rule *rule = flow_cls_offload_flow_rule(f);
+       struct flow_match_ports_range match;
+       u32 key_mask_value = 0;
+
+       if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS_RANGE))
+               return 0;
+
+       if (ip_proto != IPPROTO_TCP && ip_proto != IPPROTO_UDP) {
+               NL_SET_ERR_MSG_MOD(f->common.extack, "Only UDP and TCP keys are supported");
+               return -EINVAL;
+       }
+
+       flow_rule_match_ports_range(rule, &match);
+
+       if (match.mask->tp_min.src) {
+               struct mlxsw_sp_port_range range = {
+                       .min = ntohs(match.key->tp_min.src),
+                       .max = ntohs(match.key->tp_max.src),
+                       .source = true,
+               };
+               u8 prr_index;
+               int err;
+
+               err = mlxsw_sp_port_range_reg_get(mlxsw_sp, &range,
+                                                 f->common.extack, &prr_index);
+               if (err)
+                       return err;
+
+               rulei->src_port_range_reg_index = prr_index;
+               rulei->src_port_range_reg_valid = true;
+               key_mask_value |= BIT(prr_index);
+       }
+
+       if (match.mask->tp_min.dst) {
+               struct mlxsw_sp_port_range range = {
+                       .min = ntohs(match.key->tp_min.dst),
+                       .max = ntohs(match.key->tp_max.dst),
+               };
+               u8 prr_index;
+               int err;
+
+               err = mlxsw_sp_port_range_reg_get(mlxsw_sp, &range,
+                                                 f->common.extack, &prr_index);
+               if (err)
+                       return err;
+
+               rulei->dst_port_range_reg_index = prr_index;
+               rulei->dst_port_range_reg_valid = true;
+               key_mask_value |= BIT(prr_index);
+       }
+
+       mlxsw_sp_acl_rulei_keymask_u32(rulei, MLXSW_AFK_ELEMENT_L4_PORT_RANGE,
+                                      key_mask_value, key_mask_value);
+
+       return 0;
+}
+
 static int mlxsw_sp_flower_parse_tcp(struct mlxsw_sp *mlxsw_sp,
                                     struct mlxsw_sp_acl_rule_info *rulei,
                                     struct flow_cls_offload *f,
@@ -503,6 +565,7 @@ static int mlxsw_sp_flower_parse(struct mlxsw_sp *mlxsw_sp,
              BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
              BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
              BIT(FLOW_DISSECTOR_KEY_PORTS) |
+             BIT(FLOW_DISSECTOR_KEY_PORTS_RANGE) |
              BIT(FLOW_DISSECTOR_KEY_TCP) |
              BIT(FLOW_DISSECTOR_KEY_IP) |
              BIT(FLOW_DISSECTOR_KEY_VLAN))) {
@@ -604,6 +667,11 @@ static int mlxsw_sp_flower_parse(struct mlxsw_sp *mlxsw_sp,
        err = mlxsw_sp_flower_parse_ports(mlxsw_sp, rulei, f, ip_proto);
        if (err)
                return err;
+
+       err = mlxsw_sp_flower_parse_ports_range(mlxsw_sp, rulei, f, ip_proto);
+       if (err)
+               return err;
+
        err = mlxsw_sp_flower_parse_tcp(mlxsw_sp, rulei, f, ip_proto);
        if (err)
                return err;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_port_range.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_port_range.c
new file mode 100644 (file)
index 0000000..2d193de
--- /dev/null
@@ -0,0 +1,200 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
+/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#include <linux/bits.h>
+#include <linux/netlink.h>
+#include <linux/refcount.h>
+#include <linux/xarray.h>
+#include <net/devlink.h>
+
+#include "spectrum.h"
+
+struct mlxsw_sp_port_range_reg {
+       struct mlxsw_sp_port_range range;
+       refcount_t refcount;
+       u32 index;
+};
+
+struct mlxsw_sp_port_range_core {
+       struct xarray prr_xa;
+       struct xa_limit prr_ids;
+       atomic_t prr_count;
+};
+
+static int
+mlxsw_sp_port_range_reg_configure(struct mlxsw_sp *mlxsw_sp,
+                                 const struct mlxsw_sp_port_range_reg *prr)
+{
+       char pprr_pl[MLXSW_REG_PPRR_LEN];
+
+       /* We do not care if packet is IPv4/IPv6 and TCP/UDP, so set all four
+        * fields.
+        */
+       mlxsw_reg_pprr_pack(pprr_pl, prr->index);
+       mlxsw_reg_pprr_ipv4_set(pprr_pl, true);
+       mlxsw_reg_pprr_ipv6_set(pprr_pl, true);
+       mlxsw_reg_pprr_src_set(pprr_pl, prr->range.source);
+       mlxsw_reg_pprr_dst_set(pprr_pl, !prr->range.source);
+       mlxsw_reg_pprr_tcp_set(pprr_pl, true);
+       mlxsw_reg_pprr_udp_set(pprr_pl, true);
+       mlxsw_reg_pprr_port_range_min_set(pprr_pl, prr->range.min);
+       mlxsw_reg_pprr_port_range_max_set(pprr_pl, prr->range.max);
+
+       return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pprr), pprr_pl);
+}
+
+static struct mlxsw_sp_port_range_reg *
+mlxsw_sp_port_range_reg_create(struct mlxsw_sp *mlxsw_sp,
+                              const struct mlxsw_sp_port_range *range,
+                              struct netlink_ext_ack *extack)
+{
+       struct mlxsw_sp_port_range_core *pr_core = mlxsw_sp->pr_core;
+       struct mlxsw_sp_port_range_reg *prr;
+       int err;
+
+       prr = kzalloc(sizeof(*prr), GFP_KERNEL);
+       if (!prr)
+               return ERR_PTR(-ENOMEM);
+
+       prr->range = *range;
+       refcount_set(&prr->refcount, 1);
+
+       err = xa_alloc(&pr_core->prr_xa, &prr->index, prr, pr_core->prr_ids,
+                      GFP_KERNEL);
+       if (err) {
+               if (err == -EBUSY)
+                       NL_SET_ERR_MSG_MOD(extack, "Exceeded number of port range registers");
+               goto err_xa_alloc;
+       }
+
+       err = mlxsw_sp_port_range_reg_configure(mlxsw_sp, prr);
+       if (err) {
+               NL_SET_ERR_MSG_MOD(extack, "Failed to configure port range register");
+               goto err_reg_configure;
+       }
+
+       atomic_inc(&pr_core->prr_count);
+
+       return prr;
+
+err_reg_configure:
+       xa_erase(&pr_core->prr_xa, prr->index);
+err_xa_alloc:
+       kfree(prr);
+       return ERR_PTR(err);
+}
+
+static void mlxsw_sp_port_range_reg_destroy(struct mlxsw_sp *mlxsw_sp,
+                                           struct mlxsw_sp_port_range_reg *prr)
+{
+       struct mlxsw_sp_port_range_core *pr_core = mlxsw_sp->pr_core;
+
+       atomic_dec(&pr_core->prr_count);
+       xa_erase(&pr_core->prr_xa, prr->index);
+       kfree(prr);
+}
+
+static struct mlxsw_sp_port_range_reg *
+mlxsw_sp_port_range_reg_find(struct mlxsw_sp *mlxsw_sp,
+                            const struct mlxsw_sp_port_range *range)
+{
+       struct mlxsw_sp_port_range_core *pr_core = mlxsw_sp->pr_core;
+       struct mlxsw_sp_port_range_reg *prr;
+       unsigned long index;
+
+       xa_for_each(&pr_core->prr_xa, index, prr) {
+               if (prr->range.min == range->min &&
+                   prr->range.max == range->max &&
+                   prr->range.source == range->source)
+                       return prr;
+       }
+
+       return NULL;
+}
+
+int mlxsw_sp_port_range_reg_get(struct mlxsw_sp *mlxsw_sp,
+                               const struct mlxsw_sp_port_range *range,
+                               struct netlink_ext_ack *extack,
+                               u8 *p_prr_index)
+{
+       struct mlxsw_sp_port_range_reg *prr;
+
+       prr = mlxsw_sp_port_range_reg_find(mlxsw_sp, range);
+       if (prr) {
+               refcount_inc(&prr->refcount);
+               *p_prr_index = prr->index;
+               return 0;
+       }
+
+       prr = mlxsw_sp_port_range_reg_create(mlxsw_sp, range, extack);
+       if (IS_ERR(prr))
+               return PTR_ERR(prr);
+
+       *p_prr_index = prr->index;
+
+       return 0;
+}
+
+void mlxsw_sp_port_range_reg_put(struct mlxsw_sp *mlxsw_sp, u8 prr_index)
+{
+       struct mlxsw_sp_port_range_core *pr_core = mlxsw_sp->pr_core;
+       struct mlxsw_sp_port_range_reg *prr;
+
+       prr = xa_load(&pr_core->prr_xa, prr_index);
+       if (WARN_ON(!prr))
+               return;
+
+       if (!refcount_dec_and_test(&prr->refcount))
+               return;
+
+       mlxsw_sp_port_range_reg_destroy(mlxsw_sp, prr);
+}
+
+static u64 mlxsw_sp_port_range_reg_occ_get(void *priv)
+{
+       struct mlxsw_sp_port_range_core *pr_core = priv;
+
+       return atomic_read(&pr_core->prr_count);
+}
+
+int mlxsw_sp_port_range_init(struct mlxsw_sp *mlxsw_sp)
+{
+       struct mlxsw_sp_port_range_core *pr_core;
+       struct mlxsw_core *core = mlxsw_sp->core;
+       u64 max;
+
+       if (!MLXSW_CORE_RES_VALID(core, ACL_MAX_L4_PORT_RANGE))
+               return -EIO;
+       max = MLXSW_CORE_RES_GET(core, ACL_MAX_L4_PORT_RANGE);
+
+       /* Each port range register is represented using a single bit in the
+        * two bytes "l4_port_range" ACL key element.
+        */
+       WARN_ON(max > BITS_PER_BYTE * sizeof(u16));
+
+       pr_core = kzalloc(sizeof(*mlxsw_sp->pr_core), GFP_KERNEL);
+       if (!pr_core)
+               return -ENOMEM;
+       mlxsw_sp->pr_core = pr_core;
+
+       pr_core->prr_ids.max = max - 1;
+       xa_init_flags(&pr_core->prr_xa, XA_FLAGS_ALLOC);
+
+       devl_resource_occ_get_register(priv_to_devlink(core),
+                                      MLXSW_SP_RESOURCE_PORT_RANGE_REGISTERS,
+                                      mlxsw_sp_port_range_reg_occ_get,
+                                      pr_core);
+
+       return 0;
+}
+
+void mlxsw_sp_port_range_fini(struct mlxsw_sp *mlxsw_sp)
+{
+       struct mlxsw_sp_port_range_core *pr_core = mlxsw_sp->pr_core;
+
+       devl_resource_occ_get_unregister(priv_to_devlink(mlxsw_sp->core),
+                                        MLXSW_SP_RESOURCE_PORT_RANGE_REGISTERS);
+       WARN_ON(!xa_empty(&pr_core->prr_xa));
+       xa_destroy(&pr_core->prr_xa);
+       kfree(pr_core);
+}
diff --git a/tools/testing/selftests/drivers/net/mlxsw/port_range_occ.sh b/tools/testing/selftests/drivers/net/mlxsw/port_range_occ.sh
new file mode 100755 (executable)
index 0000000..b1f0781
--- /dev/null
@@ -0,0 +1,111 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Test that filters that match on the same port range, but with different
+# combination of IPv4/IPv6 and TCP/UDP all use the same port range register by
+# observing port range registers' occupancy via devlink-resource.
+
+lib_dir=$(dirname $0)/../../../net/forwarding
+
+ALL_TESTS="
+       port_range_occ_test
+"
+NUM_NETIFS=2
+source $lib_dir/lib.sh
+source $lib_dir/devlink_lib.sh
+
+h1_create()
+{
+       simple_if_init $h1
+}
+
+h1_destroy()
+{
+       simple_if_fini $h1
+}
+
+switch_create()
+{
+       simple_if_init $swp1
+       tc qdisc add dev $swp1 clsact
+}
+
+switch_destroy()
+{
+       tc qdisc del dev $swp1 clsact
+       simple_if_fini $swp1
+}
+
+setup_prepare()
+{
+       h1=${NETIFS[p1]}
+       swp1=${NETIFS[p2]}
+
+       vrf_prepare
+
+       h1_create
+       switch_create
+}
+
+cleanup()
+{
+       pre_cleanup
+
+       switch_destroy
+       h1_destroy
+
+       vrf_cleanup
+}
+
+port_range_occ_get()
+{
+       devlink_resource_occ_get port_range_registers
+}
+
+port_range_occ_test()
+{
+       RET=0
+
+       local occ=$(port_range_occ_get)
+
+       # Two port range registers are used, for source and destination port
+       # ranges.
+       tc filter add dev $swp1 ingress pref 1 handle 101 proto ip \
+               flower skip_sw ip_proto udp src_port 1-100 dst_port 1-100 \
+               action pass
+       (( occ + 2 == $(port_range_occ_get) ))
+       check_err $? "Got occupancy $(port_range_occ_get), expected $((occ + 2))"
+
+       tc filter add dev $swp1 ingress pref 1 handle 102 proto ip \
+               flower skip_sw ip_proto tcp src_port 1-100 dst_port 1-100 \
+               action pass
+       tc filter add dev $swp1 ingress pref 2 handle 103 proto ipv6 \
+               flower skip_sw ip_proto udp src_port 1-100 dst_port 1-100 \
+               action pass
+       tc filter add dev $swp1 ingress pref 2 handle 104 proto ipv6 \
+               flower skip_sw ip_proto tcp src_port 1-100 dst_port 1-100 \
+               action pass
+       (( occ + 2 == $(port_range_occ_get) ))
+       check_err $? "Got occupancy $(port_range_occ_get), expected $((occ + 2))"
+
+       tc filter del dev $swp1 ingress pref 2 handle 104 flower
+       tc filter del dev $swp1 ingress pref 2 handle 103 flower
+       tc filter del dev $swp1 ingress pref 1 handle 102 flower
+       (( occ + 2 == $(port_range_occ_get) ))
+       check_err $? "Got occupancy $(port_range_occ_get), expected $((occ + 2))"
+
+       tc filter del dev $swp1 ingress pref 1 handle 101 flower
+       (( occ == $(port_range_occ_get) ))
+       check_err $? "Got occupancy $(port_range_occ_get), expected $occ"
+
+       log_test "port range occupancy"
+}
+
+trap cleanup EXIT
+
+setup_prepare
+setup_wait
+
+tests_run
+
+exit $EXIT_STATUS
diff --git a/tools/testing/selftests/drivers/net/mlxsw/port_range_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/port_range_scale.sh
new file mode 100644 (file)
index 0000000..2a70840
--- /dev/null
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: GPL-2.0
+
+PORT_RANGE_NUM_NETIFS=2
+
+port_range_h1_create()
+{
+       simple_if_init $h1
+}
+
+port_range_h1_destroy()
+{
+       simple_if_fini $h1
+}
+
+port_range_switch_create()
+{
+       simple_if_init $swp1
+       tc qdisc add dev $swp1 clsact
+}
+
+port_range_switch_destroy()
+{
+       tc qdisc del dev $swp1 clsact
+       simple_if_fini $swp1
+}
+
+port_range_rules_create()
+{
+       local count=$1; shift
+       local should_fail=$1; shift
+       local batch_file="$(mktemp)"
+
+       for ((i = 0; i < count; ++i)); do
+               cat >> $batch_file <<-EOF
+                       filter add dev $swp1 ingress \
+                               prot ipv4 \
+                               pref 1000 \
+                               flower skip_sw \
+                               ip_proto udp dst_port 1-$((100 + i)) \
+                               action pass
+               EOF
+       done
+
+       tc -b $batch_file
+       check_err_fail $should_fail $? "Rule insertion"
+
+       rm -f $batch_file
+}
+
+__port_range_test()
+{
+       local count=$1; shift
+       local should_fail=$1; shift
+
+       port_range_rules_create $count $should_fail
+
+       offload_count=$(tc -j filter show dev $swp1 ingress |
+                       jq "[.[] | select(.options.in_hw == true)] | length")
+       ((offload_count == count))
+       check_err_fail $should_fail $? "port range offload count"
+}
+
+port_range_test()
+{
+       local count=$1; shift
+       local should_fail=$1; shift
+
+       if ! tc_offload_check $PORT_RANGE_NUM_NETIFS; then
+               check_err 1 "Could not test offloaded functionality"
+               return
+       fi
+
+       __port_range_test $count $should_fail
+}
+
+port_range_setup_prepare()
+{
+       h1=${NETIFS[p1]}
+       swp1=${NETIFS[p2]}
+
+       vrf_prepare
+
+       port_range_h1_create
+       port_range_switch_create
+}
+
+port_range_cleanup()
+{
+       pre_cleanup
+
+       port_range_switch_destroy
+       port_range_h1_destroy
+
+       vrf_cleanup
+}
diff --git a/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/port_range_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/port_range_scale.sh
new file mode 120000 (symlink)
index 0000000..bd670d9
--- /dev/null
@@ -0,0 +1 @@
+../spectrum/port_range_scale.sh
\ No newline at end of file
index 688338b..a88d8a8 100755 (executable)
@@ -33,6 +33,7 @@ ALL_TESTS="
        port
        rif_mac_profile
        rif_counter
+       port_range
 "
 
 for current_test in ${TESTS:-$ALL_TESTS}; do
diff --git a/tools/testing/selftests/drivers/net/mlxsw/spectrum/port_range_scale.sh b/tools/testing/selftests/drivers/net/mlxsw/spectrum/port_range_scale.sh
new file mode 100644 (file)
index 0000000..d0847e8
--- /dev/null
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0
+source ../port_range_scale.sh
+
+port_range_get_target()
+{
+       local should_fail=$1; shift
+       local target
+
+       target=$(devlink_resource_size_get port_range_registers)
+
+       if ((! should_fail)); then
+               echo $target
+       else
+               echo $((target + 1))
+       fi
+}
index 95d9f71..f981c95 100755 (executable)
@@ -30,6 +30,7 @@ ALL_TESTS="
        port
        rif_mac_profile
        rif_counter
+       port_range
 "
 
 for current_test in ${TESTS:-$ALL_TESTS}; do
index 770efbe..f49c6c3 100644 (file)
@@ -85,6 +85,7 @@ TEST_PROGS = bridge_igmp.sh \
        tc_flower.sh \
        tc_flower_l2_miss.sh \
        tc_flower_cfm.sh \
+       tc_flower_port_range.sh \
        tc_mpls_l2vpn.sh \
        tc_police.sh \
        tc_shblocks.sh \
diff --git a/tools/testing/selftests/net/forwarding/tc_flower_port_range.sh b/tools/testing/selftests/net/forwarding/tc_flower_port_range.sh
new file mode 100755 (executable)
index 0000000..3885a2a
--- /dev/null
@@ -0,0 +1,228 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+# +-----------------------+                             +----------------------+
+# | H1 (vrf)              |                             | H2 (vrf)             |
+# |    + $h1              |                             |              $h2 +   |
+# |    | 192.0.2.1/28     |                             |     192.0.2.2/28 |   |
+# |    | 2001:db8:1::1/64 |                             | 2001:db8:1::2/64 |   |
+# +----|------------------+                             +------------------|---+
+#      |                                                                   |
+# +----|-------------------------------------------------------------------|---+
+# | SW |                                                                   |   |
+# |  +-|-------------------------------------------------------------------|-+ |
+# |  | + $swp1                       BR                              $swp2 + | |
+# |  +-----------------------------------------------------------------------+ |
+# +----------------------------------------------------------------------------+
+
+ALL_TESTS="
+       test_port_range_ipv4_udp
+       test_port_range_ipv4_tcp
+       test_port_range_ipv6_udp
+       test_port_range_ipv6_tcp
+"
+
+NUM_NETIFS=4
+source lib.sh
+source tc_common.sh
+
+h1_create()
+{
+       simple_if_init $h1 192.0.2.1/28 2001:db8:1::1/64
+}
+
+h1_destroy()
+{
+       simple_if_fini $h1 192.0.2.1/28 2001:db8:1::1/64
+}
+
+h2_create()
+{
+       simple_if_init $h2 192.0.2.2/28 2001:db8:1::2/64
+}
+
+h2_destroy()
+{
+       simple_if_fini $h2 192.0.2.2/28 2001:db8:1::2/64
+}
+
+switch_create()
+{
+       ip link add name br1 type bridge
+       ip link set dev $swp1 master br1
+       ip link set dev $swp1 up
+       ip link set dev $swp2 master br1
+       ip link set dev $swp2 up
+       ip link set dev br1 up
+
+       tc qdisc add dev $swp1 clsact
+       tc qdisc add dev $swp2 clsact
+}
+
+switch_destroy()
+{
+       tc qdisc del dev $swp2 clsact
+       tc qdisc del dev $swp1 clsact
+
+       ip link set dev br1 down
+       ip link set dev $swp2 down
+       ip link set dev $swp2 nomaster
+       ip link set dev $swp1 down
+       ip link set dev $swp1 nomaster
+       ip link del dev br1
+}
+
+__test_port_range()
+{
+       local proto=$1; shift
+       local ip_proto=$1; shift
+       local sip=$1; shift
+       local dip=$1; shift
+       local mode=$1; shift
+       local name=$1; shift
+       local dmac=$(mac_get $h2)
+       local smac=$(mac_get $h1)
+       local sport_min=100
+       local sport_max=200
+       local sport_mid=$((sport_min + (sport_max - sport_min) / 2))
+       local dport_min=300
+       local dport_max=400
+       local dport_mid=$((dport_min + (dport_max - dport_min) / 2))
+
+       RET=0
+
+       tc filter add dev $swp1 ingress protocol $proto handle 101 pref 1 \
+               flower src_ip $sip dst_ip $dip ip_proto $ip_proto \
+               src_port $sport_min-$sport_max \
+               dst_port $dport_min-$dport_max \
+               action pass
+       tc filter add dev $swp2 egress protocol $proto handle 101 pref 1 \
+               flower src_ip $sip dst_ip $dip ip_proto $ip_proto \
+               src_port $sport_min-$sport_max \
+               dst_port $dport_min-$dport_max \
+               action drop
+
+       $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \
+               -t $ip_proto "sp=$sport_min,dp=$dport_min"
+       tc_check_packets "dev $swp1 ingress" 101 1
+       check_err $? "Ingress filter not hit with minimum ports"
+       tc_check_packets "dev $swp2 egress" 101 1
+       check_err $? "Egress filter not hit with minimum ports"
+
+       $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \
+               -t $ip_proto "sp=$sport_mid,dp=$dport_mid"
+       tc_check_packets "dev $swp1 ingress" 101 2
+       check_err $? "Ingress filter not hit with middle ports"
+       tc_check_packets "dev $swp2 egress" 101 2
+       check_err $? "Egress filter not hit with middle ports"
+
+       $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \
+               -t $ip_proto "sp=$sport_max,dp=$dport_max"
+       tc_check_packets "dev $swp1 ingress" 101 3
+       check_err $? "Ingress filter not hit with maximum ports"
+       tc_check_packets "dev $swp2 egress" 101 3
+       check_err $? "Egress filter not hit with maximum ports"
+
+       # Send traffic when both ports are out of range and when only one port
+       # is out of range.
+       $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \
+               -t $ip_proto "sp=$((sport_min - 1)),dp=$dport_min"
+       $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \
+               -t $ip_proto "sp=$((sport_max + 1)),dp=$dport_min"
+       $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \
+               -t $ip_proto "sp=$sport_min,dp=$((dport_min - 1))"
+       $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \
+               -t $ip_proto "sp=$sport_min,dp=$((dport_max + 1))"
+       $MZ $mode $h1 -c 1 -q -p 100 -a $smac -b $dmac -A $sip -B $dip \
+               -t $ip_proto "sp=$((sport_max + 1)),dp=$((dport_max + 1))"
+       tc_check_packets "dev $swp1 ingress" 101 3
+       check_err $? "Ingress filter was hit when should not"
+       tc_check_packets "dev $swp2 egress" 101 3
+       check_err $? "Egress filter was hit when should not"
+
+       tc filter del dev $swp2 egress protocol $proto pref 1 handle 101 flower
+       tc filter del dev $swp1 ingress protocol $proto pref 1 handle 101 flower
+
+       log_test "Port range matching - $name"
+}
+
+test_port_range_ipv4_udp()
+{
+       local proto=ipv4
+       local ip_proto=udp
+       local sip=192.0.2.1
+       local dip=192.0.2.2
+       local mode="-4"
+       local name="IPv4 UDP"
+
+       __test_port_range $proto $ip_proto $sip $dip $mode "$name"
+}
+
+test_port_range_ipv4_tcp()
+{
+       local proto=ipv4
+       local ip_proto=tcp
+       local sip=192.0.2.1
+       local dip=192.0.2.2
+       local mode="-4"
+       local name="IPv4 TCP"
+
+       __test_port_range $proto $ip_proto $sip $dip $mode "$name"
+}
+
+test_port_range_ipv6_udp()
+{
+       local proto=ipv6
+       local ip_proto=udp
+       local sip=2001:db8:1::1
+       local dip=2001:db8:1::2
+       local mode="-6"
+       local name="IPv6 UDP"
+
+       __test_port_range $proto $ip_proto $sip $dip $mode "$name"
+}
+
+test_port_range_ipv6_tcp()
+{
+       local proto=ipv6
+       local ip_proto=tcp
+       local sip=2001:db8:1::1
+       local dip=2001:db8:1::2
+       local mode="-6"
+       local name="IPv6 TCP"
+
+       __test_port_range $proto $ip_proto $sip $dip $mode "$name"
+}
+
+setup_prepare()
+{
+       h1=${NETIFS[p1]}
+       swp1=${NETIFS[p2]}
+
+       swp2=${NETIFS[p3]}
+       h2=${NETIFS[p4]}
+
+       vrf_prepare
+       h1_create
+       h2_create
+       switch_create
+}
+
+cleanup()
+{
+       pre_cleanup
+
+       switch_destroy
+       h2_destroy
+       h1_destroy
+       vrf_cleanup
+}
+
+trap cleanup EXIT
+
+setup_prepare
+setup_wait
+
+tests_run
+
+exit $EXIT_STATUS