ovn-northd ipam: Support 'exclude_ips' option

If the CMS wants to make use of ovn ipam it can now provide a
list of IPv4 addresses and a range of IPv4 addresses which
will be excluded from the dynamic address assignment.
To support this, a new option 'exclude_ips' is added in the
Logical_switch.other_config column.

Eg. ovn-nbctl set Logical_switch sw0
other_config:exclude_ips="10.0.0.2 10.0.0.30..10.0.0.40"

The present code, uses hash maps to store the assigned IP addresses.
In order to support this option, this patch has refactored the IPAM
assignment. It now uses a bitmap to manage the IP assignment with
each bit in the bitmap representing an IPv4 address.

This patch also clears the 'Logical_switch_port.dynamic_addresses'
if the CMS has cleared 'dynamic' address assignment request.

Signed-off-by: Numan Siddique <nusiddiq@redhat.com>
Signed-off-by: Ben Pfaff <blp@ovn.org>

NEWS

@@ -13,6 +13,7 @@ Post-v2.7.0
    - New support for multiple VLANs (802.1ad or "QinQ"), including a new
      "dot1q-tunnel" port VLAN mode.
    - OVN:
+     * IPAM for IPv4 can now exclude user-defined addresses from assignment.
      * Make the DHCPv4 router setting optional.
      * Gratuitous ARP for NAT addresses on a distributed logical router.
      * Allow ovn-controller SSL configuration to be obtained from vswitchd

ovn/northd/ovn-northd.c

@@ -372,6 +372,13 @@ port_has_qos_params(const struct smap *opts)
             smap_get(opts, "qos_burst"));
 }
 
+
+struct ipam_info {
+    uint32_t start_ipv4;
+    size_t total_ipv4s;
+    unsigned long *allocated_ipv4s; /* A bitmap of allocated IPv4s */
+};
+
 /* The 'key' comes from nbs->header_.uuid or nbr->header_.uuid or
  * sb->external_ids:logical-switch. */
 struct ovn_datapath {
@@ -394,7 +401,7 @@ struct ovn_datapath {
     bool has_unknown;
 
     /* IPAM data. */
-    struct hmap ipam;
+    struct ipam_info *ipam_info;
 
     /* OVN northd only needs to know about the logical router gateway port for
      * NAT on a distributed router.  This "distributed gateway port" is
@@ -420,21 +427,6 @@ cleanup_macam(struct hmap *macam)
     }
 }
 
-struct ipam_node {
-    struct hmap_node hmap_node;
-    uint32_t ip_addr; /* Allocated IP address. */
-};
-
-static void
-destroy_ipam(struct hmap *ipam)
-{
-    struct ipam_node *node;
-    HMAP_FOR_EACH_POP (node, hmap_node, ipam) {
-        free(node);
-    }
-    hmap_destroy(ipam);
-}
-
 static struct ovn_datapath *
 ovn_datapath_create(struct hmap *datapaths, const struct uuid *key,
                     const struct nbrec_logical_switch *nbs,
@@ -447,7 +439,6 @@ ovn_datapath_create(struct hmap *datapaths, const struct uuid *key,
     od->nbs = nbs;
     od->nbr = nbr;
     hmap_init(&od->port_tnlids);
-    hmap_init(&od->ipam);
     od->port_key_hint = 0;
     hmap_insert(datapaths, &od->key_node, uuid_hash(&od->key));
     return od;
@@ -462,7 +453,10 @@ ovn_datapath_destroy(struct hmap *datapaths, struct ovn_datapath *od)
          * use it. */
         hmap_remove(datapaths, &od->key_node);
         destroy_tnlids(&od->port_tnlids);
-        destroy_ipam(&od->ipam);
+        if (od->ipam_info) {
+            bitmap_free(od->ipam_info->allocated_ipv4s);
+            free(od->ipam_info);
+        }
         free(od->router_ports);
         free(od);
     }
@@ -507,6 +501,87 @@ lrouter_is_enabled(const struct nbrec_logical_router *lrouter)
     return !lrouter->enabled || *lrouter->enabled;
 }
 
+static void
+init_ipam_info_for_datapath(struct ovn_datapath *od)
+{
+    if (!od->nbs) {
+        return;
+    }
+
+    const char *subnet_str = smap_get(&od->nbs->other_config, "subnet");
+    if (!subnet_str) {
+        return;
+    }
+
+    ovs_be32 subnet, mask;
+    char *error = ip_parse_masked(subnet_str, &subnet, &mask);
+    if (error || mask == OVS_BE32_MAX || !ip_is_cidr(mask)) {
+        static struct vlog_rate_limit rl
+            = VLOG_RATE_LIMIT_INIT(5, 1);
+        VLOG_WARN_RL(&rl, "bad 'subnet' %s", subnet_str);
+        free(error);
+        return;
+    }
+
+    od->ipam_info = xzalloc(sizeof *od->ipam_info);
+    od->ipam_info->start_ipv4 = ntohl(subnet) + 1;
+    od->ipam_info->total_ipv4s = ~ntohl(mask);
+    od->ipam_info->allocated_ipv4s =
+        bitmap_allocate(od->ipam_info->total_ipv4s);
+
+    /* Mark first IP as taken */
+    bitmap_set1(od->ipam_info->allocated_ipv4s, 0);
+
+    /* Check if there are any reserver IPs (list) to be excluded from IPAM */
+    const char *exclude_ip_list = smap_get(&od->nbs->other_config,
+                                           "exclude_ips");
+    if (!exclude_ip_list) {
+        return;
+    }
+
+    struct lexer lexer;
+    lexer_init(&lexer, exclude_ip_list);
+    /* exclude_ip_list could be in the format -
+    *  "10.0.0.4 10.0.0.10 10.0.0.20..10.0.0.50 10.0.0.100..10.0.0.110".
+    */
+    lexer_get(&lexer);
+    while (lexer.token.type != LEX_T_END) {
+        if (lexer.token.type != LEX_T_INTEGER) {
+            lexer_syntax_error(&lexer, "expecting address");
+            break;
+        }
+        uint32_t start = ntohl(lexer.token.value.ipv4);
+        lexer_get(&lexer);
+
+        uint32_t end = start + 1;
+        if (lexer_match(&lexer, LEX_T_ELLIPSIS)) {
+            if (lexer.token.type != LEX_T_INTEGER) {
+                lexer_syntax_error(&lexer, "expecting address range");
+                break;
+            }
+            end = ntohl(lexer.token.value.ipv4) + 1;
+            lexer_get(&lexer);
+        }
+
+        /* Clamp start...end to fit the subnet. */
+        start = MAX(od->ipam_info->start_ipv4, start);
+        end = MIN(od->ipam_info->start_ipv4 + od->ipam_info->total_ipv4s, end);
+        if (end > start) {
+            bitmap_set_multiple(od->ipam_info->allocated_ipv4s,
+                                start - od->ipam_info->start_ipv4,
+                                end - start, 1);
+        } else {
+            lexer_error(&lexer, "excluded addresses not in subnet");
+        }
+    }
+    if (lexer.error) {
+        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
+        VLOG_WARN_RL(&rl, "logical switch "UUID_FMT": bad exclude_ips (%s)",
+                     UUID_ARGS(&od->key), lexer.error);
+    }
+    lexer_destroy(&lexer);
+}
+
 static void
 join_datapaths(struct northd_context *ctx, struct hmap *datapaths,
                struct ovs_list *sb_only, struct ovs_list *nb_only,
@@ -560,6 +635,8 @@ join_datapaths(struct northd_context *ctx, struct hmap *datapaths,
                                      nbs, NULL, NULL);
             ovs_list_push_back(nb_only, &od->list);
         }
+
+        init_ipam_info_for_datapath(od);
     }
 
     const struct nbrec_logical_router *nbr;
@@ -787,24 +864,6 @@ ipam_is_duplicate_mac(struct eth_addr *ea, uint64_t mac64, bool warn)
     return false;
 }
 
-static bool
-ipam_is_duplicate_ip(struct ovn_datapath *od, uint32_t ip, bool warn)
-{
-    struct ipam_node *ipam_node;
-    HMAP_FOR_EACH_WITH_HASH (ipam_node, hmap_node, hash_int(ip, 0),
-                             &od->ipam) {
-        if (ipam_node->ip_addr == ip) {
-            if (warn) {
-                static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
-                VLOG_WARN_RL(&rl, "Duplicate IP set: "IP_FMT,
-                             IP_ARGS(htonl(ip)));
-            }
-            return true;
-        }
-    }
-    return false;
-}
-
 static void
 ipam_insert_mac(struct eth_addr *ea, bool check)
 {
@@ -827,19 +886,17 @@ ipam_insert_mac(struct eth_addr *ea, bool check)
 }
 
 static void
-ipam_insert_ip(struct ovn_datapath *od, uint32_t ip, bool check)
+ipam_insert_ip(struct ovn_datapath *od, uint32_t ip)
 {
-    if (!od) {
+    if (!od || !od->ipam_info || !od->ipam_info->allocated_ipv4s) {
         return;
     }
 
-    if (check && ipam_is_duplicate_ip(od, ip, true)) {
+    if (ip >= od->ipam_info->start_ipv4 &&
-        return;
+        ip < (od->ipam_info->start_ipv4 + od->ipam_info->total_ipv4s)) {
+        bitmap_set1(od->ipam_info->allocated_ipv4s,
+                    ip - od->ipam_info->start_ipv4);
     }
-
-    struct ipam_node *new_ipam_node = xmalloc(sizeof *new_ipam_node);
-    new_ipam_node->ip_addr = ip;
-    hmap_insert(&od->ipam, &new_ipam_node->hmap_node, hash_int(ip, 0));
 }
 
 static void
@@ -861,14 +918,14 @@ ipam_insert_lsp_addresses(struct ovn_datapath *od, struct ovn_port *op,
 
     /* IP is only added to IPAM if the switch's subnet option
      * is set, whereas MAC is always added to MACAM. */
-    if (!smap_get(&od->nbs->other_config, "subnet")) {
+    if (!od->ipam_info || !od->ipam_info->allocated_ipv4s) {
         destroy_lport_addresses(&laddrs);
         return;
     }
 
     for (size_t j = 0; j < laddrs.n_ipv4_addrs; j++) {
         uint32_t ip = ntohl(laddrs.ipv4_addrs[j].addr);
-        ipam_insert_ip(od, ip, true);
+        ipam_insert_ip(od, ip);
     }
 
     destroy_lport_addresses(&laddrs);
@@ -907,7 +964,7 @@ ipam_add_port_addresses(struct ovn_datapath *od, struct ovn_port *op)
 
         for (size_t i = 0; i < lrp_networks.n_ipv4_addrs; i++) {
             uint32_t ip = ntohl(lrp_networks.ipv4_addrs[i].addr);
-            ipam_insert_ip(op->peer->od, ip, true);
+            ipam_insert_ip(op->peer->od, ip);
         }
 
         destroy_lport_addresses(&lrp_networks);
@@ -944,41 +1001,32 @@ ipam_get_unused_mac(void)
 }
 
 static uint32_t
-ipam_get_unused_ip(struct ovn_datapath *od, uint32_t subnet, uint32_t mask)
+ipam_get_unused_ip(struct ovn_datapath *od)
 {
-    if (!od) {
+    if (!od || !od->ipam_info || !od->ipam_info->allocated_ipv4s) {
         return 0;
     }
 
-    uint32_t ip = 0;
+    size_t new_ip_index = bitmap_scan(od->ipam_info->allocated_ipv4s, 0, 0,
-
+                                      od->ipam_info->total_ipv4s - 1);
-    /* Find an unused IP address in subnet. x.x.x.1 is reserved for a
+    if (new_ip_index == od->ipam_info->total_ipv4s - 1) {
-     * logical router port. */
-    for (uint32_t i = 2; i < ~mask; i++) {
-        uint32_t tentative_ip = subnet + i;
-        if (!ipam_is_duplicate_ip(od, tentative_ip, false)) {
-            ip = tentative_ip;
-            break;
-        }
-    }
-
-    if (!ip) {
         static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
         VLOG_WARN_RL( &rl, "Subnet address space has been exhausted.");
+        return 0;
     }
 
-    return ip;
+    return od->ipam_info->start_ipv4 + new_ip_index;
 }
 
 static bool
 ipam_allocate_addresses(struct ovn_datapath *od, struct ovn_port *op,
-                        const char *addrspec, ovs_be32 subnet, ovs_be32 mask)
+                        const char *addrspec)
 {
     if (!od || !op || !op->nbsp) {
         return false;
     }
 
-    uint32_t ip = ipam_get_unused_ip(od, ntohl(subnet), ntohl(mask));
+    uint32_t ip = ipam_get_unused_ip(od);
     if (!ip) {
         return false;
     }
@@ -1000,9 +1048,9 @@ ipam_allocate_addresses(struct ovn_datapath *od, struct ovn_port *op,
         check_mac = false;
     }
 
-    /* Add MAC/IP to MACAM/IPAM hmaps if both addresses were allocated
+    /* Add MAC to MACAM and IP to IPAM bitmap if both addresses were allocated
      * successfully. */
-    ipam_insert_ip(od, ip, false);
+    ipam_insert_ip(od, ip);
     ipam_insert_mac(&mac, check_mac);
 
     char *new_addr = xasprintf(ETH_ADDR_FMT" "IP_FMT,
@@ -1026,20 +1074,7 @@ build_ipam(struct hmap *datapaths, struct hmap *ports)
      * ports that have the "dynamic" keyword in their addresses column. */
     struct ovn_datapath *od;
     HMAP_FOR_EACH (od, key_node, datapaths) {
-        if (od->nbs) {
+        if (!od->nbs || !od->ipam_info || !od->ipam_info->allocated_ipv4s) {
-            const char *subnet_str = smap_get(&od->nbs->other_config,
-                                              "subnet");
-            if (!subnet_str) {
-                continue;
-            }
-
-            ovs_be32 subnet, mask;
-            char *error = ip_parse_masked(subnet_str, &subnet, &mask);
-            if (error || mask == OVS_BE32_MAX || !ip_is_cidr(mask)) {
-                static struct vlog_rate_limit rl
-                    = VLOG_RATE_LIMIT_INIT(5, 1);
-                VLOG_WARN_RL(&rl, "bad 'subnet' %s", subnet_str);
-                free(error);
             continue;
         }
 
@@ -1062,8 +1097,7 @@ build_ipam(struct hmap *datapaths, struct hmap *ports)
             for (size_t j = 0; j < nbsp->n_addresses; j++) {
                 if (is_dynamic_lsp_address(nbsp->addresses[j])
                     && !nbsp->dynamic_addresses) {
-                        if (!ipam_allocate_addresses(od, op,
+                    if (!ipam_allocate_addresses(od, op, nbsp->addresses[j])
-                                             nbsp->addresses[j], subnet, mask)
                         || !extract_lsp_addresses(nbsp->dynamic_addresses,
                                         &op->lsp_addrs[op->n_lsp_addrs])) {
                         static struct vlog_rate_limit rl
@@ -1075,6 +1109,10 @@ build_ipam(struct hmap *datapaths, struct hmap *ports)
                     break;
                 }
             }
+
+            if (!nbsp->n_addresses && nbsp->dynamic_addresses) {
+                nbrec_logical_switch_port_set_dynamic_addresses(op->nbsp,
+                                                                NULL);
             }
         }
     }

ovn/ovn-nb.xml

@@ -134,18 +134,43 @@
       QOS marking rules that apply to packets within the logical switch.
     </column>
 
-    <group title="other_config">
+    <group title="IP Address Assignment">
       <p>
-        Additional configuration options for the logical switch.
+        These options control automatic IP address management (IPAM) for ports
+        attached to the logical switch.  To enable IPAM, set <ref
+        column="other_config" key="subnet"/> and optionally <ref
+        column="other_config:exclude_ips"/>.  Then, to request dynamic address
+        assignment for a particular port, use the <code>dynamic</code> keyword
+        in the <ref table="Logical_Switch_Port" column="addresses"/> column of
+        the port's <ref table="Logical_Switch_Port"/> row.
       </p>
 
       <column name="other_config" key="subnet">
         Set this to an IPv4 subnet, e.g. <code>192.168.0.0/24</code>, to enable
         <code>ovn-northd</code> to automatically assign IP addresses within
-        that subnet.  Use the <code>dynamic</code> keyword in the <ref
+        that subnet.  
-        table="Logical_Switch_Port"/> table's <ref table="Logical_Switch_Port"
+      </column>
-        column="addresses"/> column to request dynamic address assignment for a
+
-        particular port.
+      <column name="other_config" key="exclude_ips">
+        <p>
+          To exclude some addresses from automatic IP address management, set
+          this to a list of the IPv4 addresses or <code>..</code>-delimited
+          ranges to exclude.  The addresses or ranges should be a subset of
+          those in <ref column="other_config" key="subnet"/>.
+        </p>
+        <p>
+          Whether listed or not, <code>ovn-northd</code> will never allocate
+          the first or last address in a subnet, such as 192.168.0.0 or
+          192.168.0.255 in 192.168.0.0/24.
+        </p>
+        <p>
+          Examples:
+        </p>
+        <ul>
+          <li><code>192.168.0.2 192.168.0.10</code></li>
+          <li><code>192.168.0.4 192.168.0.30..192.168.0.60 192.168.0.110..192.168.0.120</code></li>
+          <li><code>192.168.0.110..192.168.0.120 192.168.0.25..192.168.0.30 192.168.0.144</code></li>
+        </ul>
       </column>
     </group>
 

tests/ovn.at

@@ -4912,6 +4912,55 @@ AT_CHECK([ovn-nbctl get Logical-Switch-Port p31 dynamic_addresses], [0],
      ["fe:dc:ba:98:76:56 192.168.1.18"
 ])
 
+
+# Test the exclude_ips from the IPAM list
+ovn-nbctl --wait=sb set logical_switch sw0 \
+other_config:exclude_ips="192.168.1.19 192.168.1.21 192.168.1.23..192.168.1.50"
+
+ovn-nbctl --wait=sb lsp-add sw0 p32 -- lsp-set-addresses p32 \
+"dynamic"
+# 192.168.1.20 should be assigned as 192.168.1.19 is excluded.
+AT_CHECK([ovn-nbctl get Logical-Switch-Port p32 dynamic_addresses], [0],
+     ["0a:00:00:00:00:21 192.168.1.20"
+])
+
+ovn-nbctl --wait=sb lsp-add sw0 p33 -- lsp-set-addresses p33 \
+"dynamic"
+# 192.168.1.22 should be assigned as 192.168.1.21 is excluded.
+AT_CHECK([ovn-nbctl get Logical-Switch-Port p33 dynamic_addresses], [0],
+     ["0a:00:00:00:00:22 192.168.1.22"
+])
+
+ovn-nbctl --wait=sb lsp-add sw0 p34 -- lsp-set-addresses p34 \
+"dynamic"
+# 192.168.1.51 should be assigned as 192.168.1.23-192.168.1.50 is excluded.
+AT_CHECK([ovn-nbctl get Logical-Switch-Port p34 dynamic_addresses], [0],
+     ["0a:00:00:00:00:23 192.168.1.51"
+])
+
+# Now clear the exclude_ips list. 192.168.1.19 should be assigned.
+ovn-nbctl --wait=sb set Logical-switch sw0 other_config:exclude_ips="invalid"
+ovn-nbctl --wait=sb lsp-add sw0 p35 -- lsp-set-addresses p35 \
+"dynamic"
+AT_CHECK([ovn-nbctl get Logical-Switch-Port p35 dynamic_addresses], [0],
+     ["0a:00:00:00:00:24 192.168.1.19"
+])
+
+# Set invalid data in exclude_ips list. It should be ignored.
+ovn-nbctl --wait=sb set Logical-switch sw0 other_config:exclude_ips="182.168.1.30"
+ovn-nbctl --wait=sb lsp-add sw0 p36 -- lsp-set-addresses p36 \
+"dynamic"
+# 192.168.1.21 should be assigned as that's the next free one.
+AT_CHECK([ovn-nbctl get Logical-Switch-Port p36 dynamic_addresses], [0],
+     ["0a:00:00:00:00:25 192.168.1.21"
+])
+
+# Clear the dynamic addresses assignment request.
+ovn-nbctl --wait=sb clear logical_switch_port p36 addresses
+AT_CHECK([ovn-nbctl get Logical-Switch-Port p36 dynamic_addresses], [0],
+         [[[]]
+])
+
 as ovn-sb
 OVS_APP_EXIT_AND_WAIT([ovsdb-server])