dpctl: dpif: Allow viewing and configuring dp cache sizes.

This patch adds a general way of viewing/configuring datapath
cache sizes. With an implementation for the netlink interface.

The ovs-dpctl/ovs-appctl show commands will display the
current cache sizes configured:

 $ ovs-dpctl show
 system@ovs-system:
   lookups: hit:25 missed:63 lost:0
   flows: 0
   masks: hit:282 total:0 hit/pkt:3.20
   cache: hit:4 hit-rate:4.54%
   caches:
     masks-cache: size:256
   port 0: ovs-system (internal)
   port 1: br-int (internal)
   port 2: genev_sys_6081 (geneve: packet_type=ptap)
   port 3: br-ex (internal)
   port 4: eth2
   port 5: sw0p1 (internal)
   port 6: sw0p3 (internal)

A specific cache can be configured as follows:

 $ ovs-appctl dpctl/cache-set-size DP CACHE SIZE
 $ ovs-dpctl cache-set-size DP CACHE SIZE

For example to disable the cache do:

 $ ovs-dpctl cache-set-size system@ovs-system masks-cache 0
 Setting cache size successful, new size 0.

Signed-off-by: Eelco Chaudron <echaudro@redhat.com>
Acked-by: Paolo Valerio <pvalerio@redhat.com>
Acked-by: Flavio Leitner <fbl@sysclose.org>
Signed-off-by: Ilya Maximets <i.maximets@ovn.org>

NEWS

@@ -13,6 +13,9 @@ Post-v2.16.0
    - Python:
      * For SSL support, the use of the pyOpenSSL library has been replaced
        with the native 'ssl' module.
+   - ovs-dpctl and 'ovs-appctl dpctl/':
+     * New commands 'cache-get-size' and 'cache-set-size' that allows to
+       get or configure linux kernel datapath cache sizes.
 
 
 v2.16.0 - 16 Aug 2021

datapath/linux/compat/include/linux/openvswitch.h

@@ -107,7 +107,7 @@ enum ovs_datapath_attr {
 	OVS_DP_ATTR_MEGAFLOW_STATS,	/* struct ovs_dp_megaflow_stats */
 	OVS_DP_ATTR_USER_FEATURES,	/* OVS_DP_F_*  */
 	OVS_DP_ATTR_PAD,
-	OVS_DP_ATTR_PAD2,
+	OVS_DP_ATTR_MASKS_CACHE_SIZE,
 	OVS_DP_ATTR_PER_CPU_PIDS,	/* Netlink PIDS to receive upcalls */
 	__OVS_DP_ATTR_MAX
 };

lib/dpctl.c

@@ -591,6 +591,36 @@ compare_port_nos(const void *a_, const void *b_)
     return a < b ? -1 : a > b;
 }
 
+static void
+show_dpif_cache__(struct dpif *dpif, struct dpctl_params *dpctl_p)
+{
+    uint32_t nr_caches;
+    int error = dpif_cache_get_supported_levels(dpif, &nr_caches);
+
+    if (error || nr_caches == 0) {
+        return;
+    }
+
+    dpctl_print(dpctl_p, "  caches:\n");
+    for (int i = 0; i < nr_caches; i++) {
+        const char *name;
+        uint32_t size;
+
+        if (dpif_cache_get_name(dpif, i, &name) ||
+            dpif_cache_get_size(dpif, i, &size)) {
+            continue;
+        }
+        dpctl_print(dpctl_p, "    %s: size:%u\n", name, size);
+    }
+}
+
+static void
+show_dpif_cache(struct dpif *dpif, struct dpctl_params *dpctl_p)
+{
+    dpctl_print(dpctl_p, "%s:\n", dpif_name(dpif));
+    show_dpif_cache__(dpif, dpctl_p);
+}
+
 static void
 show_dpif(struct dpif *dpif, struct dpctl_params *dpctl_p)
 {
@@ -623,6 +653,8 @@ show_dpif(struct dpif *dpif, struct dpctl_params *dpctl_p)
         }
     }
 
+    show_dpif_cache__(dpif, dpctl_p);
+
     odp_port_t *port_nos = NULL;
     size_t allocated_port_nos = 0, n_port_nos = 0;
     DPIF_PORT_FOR_EACH (&dpif_port, &dump, dpif) {
@@ -2409,6 +2441,92 @@ dpctl_ct_ipf_get_status(int argc, const char *argv[],
     return error;
 }
 
+static int
+dpctl_cache_get_size(int argc, const char *argv[],
+                     struct dpctl_params *dpctl_p)
+{
+    int error;
+
+    if (argc > 1) {
+        struct dpif *dpif;
+
+        error = parsed_dpif_open(argv[1], false, &dpif);
+        if (!error) {
+            show_dpif_cache(dpif, dpctl_p);
+            dpif_close(dpif);
+        } else {
+            dpctl_error(dpctl_p, error, "Opening datapath %s failed", argv[1]);
+        }
+    } else {
+        error = dps_for_each(dpctl_p, show_dpif_cache);
+    }
+
+    return error;
+}
+
+static int
+dpctl_cache_set_size(int argc, const char *argv[],
+                     struct dpctl_params *dpctl_p)
+{
+    uint32_t nr_caches, size;
+    int i, error = EINVAL;
+    struct dpif *dpif;
+
+    if (argc != 4) {
+        dpctl_error(dpctl_p, error, "Invalid number of arguments");
+        return error;
+    }
+
+    if (!ovs_scan(argv[3], "%"SCNu32, &size)) {
+        dpctl_error(dpctl_p, error, "size is malformed");
+        return error;
+    }
+
+    error = parsed_dpif_open(argv[1], false, &dpif);
+    if (error) {
+            dpctl_error(dpctl_p, error, "Opening datapath %s failed",
+                        argv[1]);
+            return error;
+    }
+
+    error = dpif_cache_get_supported_levels(dpif, &nr_caches);
+    if (error || nr_caches == 0) {
+        dpctl_error(dpctl_p, error, "Setting caches not supported");
+        goto exit;
+    }
+
+    for (i = 0; i < nr_caches; i++) {
+        const char *name;
+
+        if (dpif_cache_get_name(dpif, i, &name)) {
+            continue;
+        }
+        if (!strcmp(argv[2], name)) {
+            break;
+        }
+    }
+
+    if (i == nr_caches) {
+        error = EINVAL;
+        dpctl_error(dpctl_p, error, "Cache name \"%s\" not found on dpif",
+                    argv[2]);
+        goto exit;
+    }
+
+    error = dpif_cache_set_size(dpif, i, size);
+    if (!error) {
+        dpif_cache_get_size(dpif, i, &size);
+        dpctl_print(dpctl_p, "Setting cache size successful, new size %u\n",
+                    size);
+    } else {
+        dpctl_error(dpctl_p, error, "Setting cache size failed");
+    }
+
+exit:
+    dpif_close(dpif);
+    return error;
+}
+
 /* Undocumented commands for unit testing. */
 
 static int
@@ -2710,6 +2828,8 @@ static const struct dpctl_command all_commands[] = {
       0, 4, dpctl_dump_conntrack, DP_RO },
     { "flush-conntrack", "[dp] [zone=N] [ct-tuple]", 0, 3,
       dpctl_flush_conntrack, DP_RW },
+    { "cache-get-size", "[dp]", 0, 1, dpctl_cache_get_size, DP_RO },
+    { "cache-set-size", "dp cache <size>", 3, 3, dpctl_cache_set_size, DP_RW },
     { "ct-stats-show", "[dp] [zone=N]",
       0, 3, dpctl_ct_stats_show, DP_RO },
     { "ct-bkts", "[dp] [gt=N]", 0, 2, dpctl_ct_bkts, DP_RO },

lib/dpctl.man

@@ -226,6 +226,20 @@ Fetches the flow from \fIdp\fR's flow table with unique identifier \fIufid\fR.
 .
 .IP "\*(DX\fBdel\-flows\fR [\fIdp\fR]"
 Deletes all flow entries from datapath \fIdp\fR's flow table.
+.SS "DATAPATH FLOW CACHE COMMANDS"
+The following commands are useful for debugging and configuring
+the datapath flow cache settings.
+.
+.TP
+\*(DX\fBcache\-get\-size\fR [\fIdp\fR]
+Prints the current cache sizes to the console.
+.
+.TP
+\*(DX\fBcache\-set\-size\fR \fIdp\fR \fIcache\fR \fIsize\fR
+Set the \fIdp\fR's specific \fIcache\fR to the given \fIsize\fR.
+The cache name can be found by using the \fBcache\-get\-size\fR
+command.
+.
 .SS "CONNECTION TRACKING TABLE COMMANDS"
 The following commands are useful for debugging and configuring
 the connection tracking table in the datapath.

lib/dpif-netdev.c

@@ -8812,6 +8812,10 @@ const struct dpif_class dpif_netdev_class = {
     dpif_netdev_bond_add,
     dpif_netdev_bond_del,
     dpif_netdev_bond_stats_get,
+    NULL,                       /* cache_get_supported_levels */
+    NULL,                       /* cache_get_name */
+    NULL,                       /* cache_get_size */
+    NULL,                       /* cache_set_size */
 };
 
 static void

lib/dpif-netlink.c

@@ -100,6 +100,7 @@ struct dpif_netlink_dp {
     const char *name;                  /* OVS_DP_ATTR_NAME. */
     const uint32_t *upcall_pid;        /* OVS_DP_ATTR_UPCALL_PID. */
     uint32_t user_features;            /* OVS_DP_ATTR_USER_FEATURES */
+    uint32_t cache_size;               /* OVS_DP_ATTR_MASKS_CACHE_SIZE */
     const struct ovs_dp_stats *stats;  /* OVS_DP_ATTR_STATS. */
     const struct ovs_dp_megaflow_stats *megaflow_stats;
                                        /* OVS_DP_ATTR_MEGAFLOW_STATS.*/
@@ -4298,6 +4299,104 @@ probe_broken_meters(struct dpif *dpif)
     }
     return broken_meters;
 }
+
+
+static int
+dpif_netlink_cache_get_supported_levels(struct dpif *dpif_, uint32_t *levels)
+{
+    struct dpif_netlink_dp dp;
+    struct ofpbuf *buf;
+    int error;
+
+    /* If available, in the kernel we support one level of cache.
+     * Unfortunately, there is no way to detect if the older kernel module has
+     * the cache feature.  For now, we only report the cache information if the
+     * kernel module reports the OVS_DP_ATTR_MASKS_CACHE_SIZE attribute. */
+
+    *levels = 0;
+    error = dpif_netlink_dp_get(dpif_, &dp, &buf);
+    if (!error) {
+
+        if (dp.cache_size != UINT32_MAX) {
+            *levels = 1;
+        }
+        ofpbuf_delete(buf);
+    }
+
+    return error;
+}
+
+static int
+dpif_netlink_cache_get_name(struct dpif *dpif_ OVS_UNUSED, uint32_t level,
+                            const char **name)
+{
+    if (level != 0) {
+        return EINVAL;
+    }
+
+    *name = "masks-cache";
+    return 0;
+}
+
+static int
+dpif_netlink_cache_get_size(struct dpif *dpif_, uint32_t level, uint32_t *size)
+{
+    struct dpif_netlink_dp dp;
+    struct ofpbuf *buf;
+    int error;
+
+    if (level != 0) {
+        return EINVAL;
+    }
+
+    error = dpif_netlink_dp_get(dpif_, &dp, &buf);
+    if (!error) {
+
+        ofpbuf_delete(buf);
+
+        if (dp.cache_size == UINT32_MAX) {
+            return EOPNOTSUPP;
+        }
+        *size = dp.cache_size;
+    }
+    return error;
+}
+
+static int
+dpif_netlink_cache_set_size(struct dpif *dpif_, uint32_t level, uint32_t size)
+{
+    struct dpif_netlink *dpif = dpif_netlink_cast(dpif_);
+    struct dpif_netlink_dp request, reply;
+    struct ofpbuf *bufp;
+    int error;
+
+    size = ROUND_UP_POW2(size);
+
+    if (level != 0) {
+        return EINVAL;
+    }
+
+    dpif_netlink_dp_init(&request);
+    request.cmd = OVS_DP_CMD_SET;
+    request.name = dpif_->base_name;
+    request.dp_ifindex = dpif->dp_ifindex;
+    request.cache_size = size;
+    /* We need to set the dpif user_features, as the kernel module assumes the
+     * OVS_DP_ATTR_USER_FEATURES attribute is always present. If not, it will
+     * reset all the features. */
+    request.user_features = dpif->user_features;
+
+    error = dpif_netlink_dp_transact(&request, &reply, &bufp);
+    if (!error) {
+        ofpbuf_delete(bufp);
+        if (reply.cache_size != size) {
+            return EINVAL;
+        }
+    }
+
+    return error;
+}
+
 
 const struct dpif_class dpif_netlink_class = {
     "system",
@@ -4377,6 +4476,10 @@ const struct dpif_class dpif_netlink_class = {
     NULL,                       /* bond_add */
     NULL,                       /* bond_del */
     NULL,                       /* bond_stats_get */
+    dpif_netlink_cache_get_supported_levels,
+    dpif_netlink_cache_get_name,
+    dpif_netlink_cache_get_size,
+    dpif_netlink_cache_set_size,
 };
 
 static int
@@ -4640,6 +4743,9 @@ dpif_netlink_dp_from_ofpbuf(struct dpif_netlink_dp *dp, const struct ofpbuf *buf
         [OVS_DP_ATTR_USER_FEATURES] = {
                         .type = NL_A_U32,
                         .optional = true },
+        [OVS_DP_ATTR_MASKS_CACHE_SIZE] = {
+                        .type = NL_A_U32,
+                        .optional = true },
     };
 
     dpif_netlink_dp_init(dp);
@@ -4672,6 +4778,12 @@ dpif_netlink_dp_from_ofpbuf(struct dpif_netlink_dp *dp, const struct ofpbuf *buf
         dp->user_features = nl_attr_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
     }
 
+    if (a[OVS_DP_ATTR_MASKS_CACHE_SIZE]) {
+        dp->cache_size = nl_attr_get_u32(a[OVS_DP_ATTR_MASKS_CACHE_SIZE]);
+    } else {
+        dp->cache_size = UINT32_MAX;
+    }
+
     return 0;
 }
 
@@ -4705,6 +4817,10 @@ dpif_netlink_dp_to_ofpbuf(const struct dpif_netlink_dp *dp, struct ofpbuf *buf)
                           sizeof *dp->upcall_pids * dp->n_upcall_pids);
     }
 
+    if (dp->cache_size != UINT32_MAX) {
+        nl_msg_put_u32(buf, OVS_DP_ATTR_MASKS_CACHE_SIZE, dp->cache_size);
+    }
+
     /* Skip OVS_DP_ATTR_STATS since we never have a reason to serialize it. */
 }
 
@@ -4713,6 +4829,7 @@ static void
 dpif_netlink_dp_init(struct dpif_netlink_dp *dp)
 {
     memset(dp, 0, sizeof *dp);
+    dp->cache_size = UINT32_MAX;
 }
 
 static void

lib/dpif-provider.h

@@ -635,6 +635,26 @@ struct dpif_class {
      * sufficient to store BOND_BUCKETS number of elements. */
     int (*bond_stats_get)(struct dpif *dpif, uint32_t bond_id,
                           uint64_t *n_bytes);
+
+    /* Cache configuration
+     *
+     * Multiple levels of cache can exist in a given datapath implementation.
+     * An API has been provided to get the number of supported caches, which
+     * can then be used to get/set specific configuration. Cache level is 0
+     * indexed, i.e. if 1 level is supported, the level value to use is 0.
+     *
+     * Get the number of cache levels supported. */
+    int (*cache_get_supported_levels)(struct dpif *dpif, uint32_t *levels);
+
+    /* Get the cache name for the given level. */
+    int (*cache_get_name)(struct dpif *dpif, uint32_t level,
+                          const char **name);
+
+    /* Get currently configured cache size. */
+    int (*cache_get_size)(struct dpif *dpif, uint32_t level, uint32_t *size);
+
+    /* Set cache size. */
+    int (*cache_set_size)(struct dpif *dpif, uint32_t level, uint32_t size);
 };
 
 extern const struct dpif_class dpif_netlink_class;

lib/dpif.c

@@ -2058,3 +2058,35 @@ dpif_get_n_offloaded_flows(struct dpif *dpif, uint64_t *n_flows)
     }
     return n_devs ? 0 : EOPNOTSUPP;
 }
+
+int
+dpif_cache_get_supported_levels(struct dpif *dpif, uint32_t *levels)
+{
+    return dpif->dpif_class->cache_get_supported_levels
+           ? dpif->dpif_class->cache_get_supported_levels(dpif, levels)
+           : EOPNOTSUPP;
+}
+
+int
+dpif_cache_get_name(struct dpif *dpif, uint32_t level, const char **name)
+{
+    return dpif->dpif_class->cache_get_name
+           ? dpif->dpif_class->cache_get_name(dpif, level, name)
+           : EOPNOTSUPP;
+}
+
+int
+dpif_cache_get_size(struct dpif *dpif, uint32_t level, uint32_t *size)
+{
+    return dpif->dpif_class->cache_get_size
+           ? dpif->dpif_class->cache_get_size(dpif, level, size)
+           : EOPNOTSUPP;
+}
+
+int
+dpif_cache_set_size(struct dpif *dpif, uint32_t level, uint32_t size)
+{
+    return dpif->dpif_class->cache_set_size
+           ? dpif->dpif_class->cache_set_size(dpif, level, size)
+           : EOPNOTSUPP;
+}

lib/dpif.h

@@ -908,6 +908,13 @@ int dpif_bond_del(struct dpif *, uint32_t bond_id);
 int dpif_bond_stats_get(struct dpif *, uint32_t bond_id, uint64_t *n_bytes);
 bool dpif_supports_lb_output_action(const struct dpif *);
 
+
+/* Cache */
+int dpif_cache_get_supported_levels(struct dpif *dpif, uint32_t *levels);
+int dpif_cache_get_name(struct dpif *dpif, uint32_t level, const char **name);
+int dpif_cache_get_size(struct dpif *dpif, uint32_t level, uint32_t *size);
+int dpif_cache_set_size(struct dpif *dpif, uint32_t level, uint32_t size);
+
 
 /* Miscellaneous. */
 

tests/system-traffic.at

@@ -1454,6 +1454,42 @@ AT_CHECK([ovs-ofctl dump-flows br0 | grep "in_port=4" | ofctl_strip], [0], [dnl
 OVS_TRAFFIC_VSWITCHD_STOP
 AT_CLEANUP
 
+AT_SETUP([datapath - configure cache size])
+
+OVS_TRAFFIC_VSWITCHD_START()
+OVS_CHECK_KERNEL_EXCL(3, 10, 5, 8)
+
+AT_CHECK([ovs-dpctl cache-get-size one-bad-dp], [1], [], [dnl
+ovs-dpctl: Opening datapath one-bad-dp failed (No such device)
+])
+AT_CHECK([ovs-dpctl cache-get-size | grep masks-cache | tr -d [[:blank:]]], [0], [dnl
+masks-cache:size:256
+])
+AT_CHECK([ovs-dpctl cache-set-size one-bad-dp masks-cache 0], [1], [], [dnl
+ovs-dpctl: Opening datapath one-bad-dp failed (No such device)
+])
+AT_CHECK([ovs-dpctl cache-set-size system@ovs-system dummy-cache 0], [1], [], [dnl
+ovs-dpctl: Cache name "dummy-cache" not found on dpif (Invalid argument)
+])
+AT_CHECK([ovs-dpctl cache-set-size system@ovs-system masks-cache 80000], [1], [], [dnl
+ovs-dpctl: Setting cache size failed (Numerical result out of range)
+])
+AT_CHECK([ovs-dpctl cache-set-size system@ovs-system masks-cache 0], [0], [dnl
+Setting cache size successful, new size 0
+])
+AT_CHECK([ovs-dpctl cache-get-size | grep masks-cache | tr -d [[:blank:]]], [0], [dnl
+masks-cache:size:0
+])
+AT_CHECK([ovs-dpctl cache-set-size system@ovs-system masks-cache 256], [0], [dnl
+Setting cache size successful, new size 256
+])
+AT_CHECK([ovs-dpctl cache-get-size | grep masks-cache | tr -d [[:blank:]]], [0], [dnl
+masks-cache:size:256
+])
+
+OVS_TRAFFIC_VSWITCHD_STOP
+AT_CLEANUP
+
 AT_BANNER([conntrack])
 
 AT_SETUP([conntrack - controller])

utilities/ovs-dpctl.c

@@ -198,6 +198,10 @@ usage(void *userdata OVS_UNUSED)
            "  del-flow [DP] FLOW         delete FLOW from DP\n"
            "  del-flows [DP] [FILE]      " \
                "delete all or specified flows from DP\n"
+           "  cache-get-size [DP]             " \
+               "Show the current size for all caches\n"
+           "  cache-set-size DP CACHE SIZE  " \
+               "Set cache size for a specific cache\n"
            "  dump-conntrack [DP] [zone=ZONE]  " \
                "display conntrack entries for ZONE\n"
            "  flush-conntrack [DP] [zone=ZONE] [ct-tuple]" \