ovsdb-idl: Add APIs to query if a table and a column is present.

This patch adds 2 new APIs in the ovsdb-idl client library
 - ovsdb_idl_server_has_table() and ovsdb_idl_server_has_column() to
query if a table and a column is present in the IDL or not.  This
patch also adds IDL helper functions which are auto generated from
the schema which makes it easier for the clients.

These APIs are required for scenarios where the server schema is old and
missing a table or column and the client (built with a new schema
version) does a transaction with the missing table or column.  This
results in a continuous loop of transaction failures.

Related-Bug: https://bugzilla.redhat.com/show_bug.cgi?id=1992705
Signed-off-by: Numan Siddique <nusiddiq@redhat.com>
Signed-off-by: Ilya Maximets <i.maximets@ovn.org>

lib/ovsdb-idl-provider.h

@@ -24,6 +24,7 @@
 #include "ovsdb-set-op.h"
 #include "ovsdb-types.h"
 #include "openvswitch/shash.h"
+#include "sset.h"
 #include "uuid.h"
 
 #ifdef __cplusplus
@@ -117,9 +118,12 @@ struct ovsdb_idl_table {
     bool need_table;         /* Monitor table even if no columns are selected
                               * for replication. */
     struct shash columns;    /* Contains "const struct ovsdb_idl_column *"s. */
+    struct sset schema_columns; /* Column names from schema. */
     struct hmap rows;        /* Contains "struct ovsdb_idl_row"s. */
     struct ovsdb_idl *idl;   /* Containing IDL instance. */
     unsigned int change_seqno[OVSDB_IDL_CHANGE_MAX];
+    bool in_server_schema;   /* Indicates if this table is in the server schema
+                              * or not. */
     struct ovs_list indexes;    /* Contains "struct ovsdb_idl_index"s */
     struct ovs_list track_list; /* Tracked rows (ovsdb_idl_row.track_node). */
 };

lib/ovsdb-idl.c

@@ -287,6 +287,8 @@ ovsdb_idl_create_unconnected(const struct ovsdb_idl_class *class,
             = table->change_seqno[OVSDB_IDL_CHANGE_MODIFY]
             = table->change_seqno[OVSDB_IDL_CHANGE_DELETE] = 0;
         table->idl = idl;
+        table->in_server_schema = false;
+        sset_init(&table->schema_columns);
     }
 
     return idl;
@@ -337,6 +339,7 @@ ovsdb_idl_destroy(struct ovsdb_idl *idl)
             struct ovsdb_idl_table *table = &idl->tables[i];
             ovsdb_idl_destroy_indexes(table);
             shash_destroy(&table->columns);
+            sset_destroy(&table->schema_columns);
             hmap_destroy(&table->rows);
             free(table->modes);
         }
@@ -718,10 +721,16 @@ ovsdb_idl_compose_monitor_request(const struct json *schema_json, void *idl_)
 
         struct json *columns
             = table->need_table ? json_array_create_empty() : NULL;
+        sset_clear(&table->schema_columns);
         for (size_t j = 0; j < tc->n_columns; j++) {
             const struct ovsdb_idl_column *column = &tc->columns[j];
             bool idl_has_column = (table_schema &&
                                   sset_contains(table_schema, column->name));
+
+            if (idl_has_column) {
+                sset_add(&table->schema_columns, column->name);
+            }
+
             if (column->is_synthetic) {
                 if (idl_has_column) {
                     VLOG_WARN("%s table in %s database has synthetic "
@@ -749,7 +758,10 @@ ovsdb_idl_compose_monitor_request(const struct json *schema_json, void *idl_)
                           "(database needs upgrade?)",
                           idl->class_->database, table->class_->name);
                 json_destroy(columns);
+                table->in_server_schema = false;
                 continue;
+            } else if (schema && table_schema) {
+                table->in_server_schema = true;
             }
 
             monitor_request = json_object_create();
@@ -790,7 +802,7 @@ ovsdb_idl_table_class_from_column(const struct ovsdb_idl_class *class,
 
 /* Given 'column' in some table in 'idl', returns the table. */
 static struct ovsdb_idl_table *
-ovsdb_idl_table_from_column(struct ovsdb_idl *idl,
+ovsdb_idl_table_from_column(const struct ovsdb_idl *idl,
                             const struct ovsdb_idl_column *column)
 {
     const struct ovsdb_idl_table_class *tc =
@@ -885,6 +897,47 @@ ovsdb_idl_add_table(struct ovsdb_idl *idl,
 
     OVS_NOT_REACHED();
 }
+
+/* Returns 'true' if the 'idl' has seen the table for the 'table_class'
+ * in the schema reported by the server.  Returns 'false' otherwise.
+ *
+ * Always returns 'false' if idl has never been connected.
+ *
+ * Please see ovsdb_idl_compose_monitor_request() which sets
+ * 'struct ovsdb_idl_table *'->in_server_schema accordingly.
+ *
+ * Usually this function is used indirectly through one of the
+ * "server_has_table" functions generated by ovsdb-idlc. */
+bool
+ovsdb_idl_server_has_table(const struct ovsdb_idl *idl,
+                           const struct ovsdb_idl_table_class *table_class)
+{
+    const struct ovsdb_idl_table *table =
+        ovsdb_idl_table_from_class(idl, table_class);
+
+    return (table && table->in_server_schema);
+}
+
+/* Returns 'true' if the 'idl' has seen the 'column' in the schema
+ * reported by the server.  Returns 'false' otherwise.
+ *
+ * Always returns 'false' if idl has never been connected.
+ *
+ * Please see ovsdb_idl_compose_monitor_request() which sets
+ * 'struct ovsdb_idl_table *'->schema_columns accordingly.
+ *
+ * Usually this function is used indirectly through one of the
+ * "server_has_column" functions generated by ovsdb-idlc. */
+bool
+ovsdb_idl_server_has_column(const struct ovsdb_idl *idl,
+                            const struct ovsdb_idl_column *column)
+{
+    const struct ovsdb_idl_table *table =
+        ovsdb_idl_table_from_column(idl, column);
+
+    return (table->in_server_schema && sset_find(&table->schema_columns,
+                                                 column->name));
+}
 
 /* A single clause within an ovsdb_idl_condition. */
 struct ovsdb_idl_clause {

lib/ovsdb-idl.h

@@ -94,6 +94,10 @@ void ovsdb_idl_check_consistency(const struct ovsdb_idl *);
 const struct ovsdb_idl_class *ovsdb_idl_get_class(const struct ovsdb_idl *);
 const struct ovsdb_idl_table_class *ovsdb_idl_table_class_from_column(
     const struct ovsdb_idl_class *, const struct ovsdb_idl_column *);
+bool ovsdb_idl_server_has_table(const struct ovsdb_idl *,
+                                const struct ovsdb_idl_table_class *);
+bool ovsdb_idl_server_has_column(const struct ovsdb_idl *,
+                                 const struct ovsdb_idl_column *);
 
 /* Choosing columns and tables to replicate.
  *

ovsdb/ovsdb-idlc.in

@@ -236,7 +236,13 @@ extern "C" {
 
         print("\nextern struct ovsdb_idl_column %s_columns[%s_N_COLUMNS];" % (structName, structName.upper()))
 
+        for columnName in table.columns:
+            print("bool %(p)sserver_has_%(t)s_table_col_%(c)s(const struct ovsdb_idl *); " % {
+                'p': prefix, 't': tableName.lower(),
+                'c': columnName})
+
         print('''
+bool %(p)sserver_has_%(t)s_table(const struct ovsdb_idl *);
 const struct %(s)s_table *%(s)s_table_get(const struct ovsdb_idl *);
 const struct %(s)s *%(s)s_table_first(const struct %(s)s_table *);
 
@@ -349,7 +355,7 @@ struct %(s)s *%(s)s_insert(struct ovsdb_idl_txn *);
  * used to test if an uncommitted row that has been added locally has been
  * updated or it may given unexpected results. */
 bool %(s)s_is_updated(const struct %(s)s *, enum %(s)s_column_id);
-''' % {'s': structName, 'S': structName.upper()})
+''' % {'s': structName, 'S': structName.upper(), 't': tableName.lower(), 'p': prefix})
 
         for columnName, column in sorted_columns(table):
             if column.extensions.get('synthetic'):
@@ -489,6 +495,26 @@ static struct %(s)s *
         print("")
         print("/* %s table. */" % (tableName))
 
+        for columnName in table.columns:
+            print('''
+bool
+%(p)sserver_has_%(t)s_table_col_%(c)s(const struct ovsdb_idl *idl)
+{
+    return ovsdb_idl_server_has_column(idl, &%(s)s_col_%(c)s);
+}
+''' % {'s': structName, 'p': prefix, 't': tableName.lower(),
+       'P': prefix.upper(), 'T': tableName.upper(),
+       'c': columnName})
+
+        print('''
+bool
+%(p)sserver_has_%(t)s_table(const struct ovsdb_idl *idl)
+{
+    return ovsdb_idl_server_has_table(idl, &%(p)stable_classes[%(P)sTABLE_%(T)s]);
+}
+''' % {'s': structName, 'p': prefix, 't': tableName.lower(),
+       'P': prefix.upper(), 'T': tableName.upper()})
+
         print('''
 const struct %(s)s_table *
 %(s)s_table_get(const struct ovsdb_idl *idl)

tests/idltest.ovsschema

@@ -210,6 +210,15 @@
       },
       "isRoot": true
     },
+    "simple7" : {
+      "columns" : {
+        "name" : {
+          "type": "string"
+        },
+        "id": {"type": "string"}
+      },
+      "isRoot" : true
+    },
     "singleton" : {
       "columns" : {
         "name" : {

tests/idltest2.ovsschema

@@ -139,6 +139,13 @@
           "type": "string"
         }
       }
+    },
+    "simple7" : {
+      "columns" : {
+        "name" : {
+          "type": "string"
+        }
+      }
     }
   }
 }

tests/ovsdb-idl.at

@@ -999,6 +999,7 @@ test-ovsdb|ovsdb_idl|idltest database lacks simple5 table (database needs upgrad
 test-ovsdb|ovsdb_idl|idltest database lacks simple6 table (database needs upgrade?)
 test-ovsdb|ovsdb_idl|idltest database lacks singleton table (database needs upgrade?)
 test-ovsdb|ovsdb_idl|link1 table in idltest database lacks l2 column (database needs upgrade?)
+test-ovsdb|ovsdb_idl|simple7 table in idltest database lacks id column (database needs upgrade?)
 ])
 
 # Check that ovsdb-idl sent on "monitor" request and that it didn't
@@ -2372,3 +2373,41 @@ OVSDB_CHECK_CLUSTER_IDL([simple idl, initially empty, force reconnect],
 [],
 [],
 reconnect.*waiting .* seconds before reconnect)
+
+AT_SETUP([idl table and column presence check])
+AT_KEYWORDS([ovsdb server idl table column check])
+AT_CHECK([ovsdb_start_idltest "" "$abs_srcdir/idltest2.ovsschema"])
+
+AT_CHECK(ovsdb-tool create db2 $abs_srcdir/idltest.ovsschema)
+AT_CHECK(ovsdb-server -vconsole:warn --log-file=ovsdb-server2.log --detach dnl
+           --no-chdir --pidfile=ovsdb-server2.pid --remote=punix:socket2 db2)
+on_exit 'kill `cat ovsdb-server2.pid`'
+
+dnl In this test, test-ovsdb first connects to the server with schema
+dnl idltest2.ovsschema and outputs the presence of tables and columns.
+dnl And then it connectes to the server with the schema idltest.ovsschema
+dnl and does the same.
+AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 dnl
+                 idl-table-column-check unix:socket unix:socket2], [0], [dnl
+unix:socket remote has table simple
+unix:socket remote has table link1
+unix:socket remote doesn't have table link2
+unix:socket remote doesn't have table simple5
+unix:socket remote doesn't have col irefmap in table simple5
+unix:socket remote doesn't have col l2 in table link1
+unix:socket remote has col i in table link1
+unix:socket remote doesn't have col id in table simple7
+--- remote unix:socket done ---
+unix:socket2 remote has table simple
+unix:socket2 remote has table link1
+unix:socket2 remote has table link2
+unix:socket2 remote has table simple5
+unix:socket2 remote has col irefmap in table simple5
+unix:socket2 remote has col l2 in table link1
+unix:socket2 remote has col i in table link1
+unix:socket2 remote has col id in table simple7
+--- remote unix:socket2 done ---
+], [stderr])
+
+OVSDB_SERVER_SHUTDOWN
+AT_CLEANUP

tests/test-ovsdb.c

@@ -3268,6 +3268,86 @@ do_idl_compound_index(struct ovs_cmdl_context *ctx)
     printf("%03d: done\n", step);
 }
 
+static void
+do_idl_table_column_check(struct ovs_cmdl_context *ctx)
+{
+    struct jsonrpc *rpc;
+    struct ovsdb_idl *idl;
+    unsigned int seqno = 0;
+    int error;
+
+    idl = ovsdb_idl_create(ctx->argv[1], &idltest_idl_class, true, true);
+    ovsdb_idl_omit(idl, &idltest_link1_col_i);
+    ovsdb_idl_omit(idl, &idltest_simple7_col_id);
+    ovsdb_idl_set_leader_only(idl, false);
+    struct stream *stream;
+
+    error = stream_open_block(jsonrpc_stream_open(ctx->argv[1], &stream,
+                              DSCP_DEFAULT), -1, &stream);
+    if (error) {
+        ovs_fatal(error, "failed to connect to \"%s\"", ctx->argv[1]);
+    }
+    rpc = jsonrpc_open(stream);
+
+    for (int r = 1; r <= 2; r++) {
+        ovsdb_idl_set_remote(idl, ctx->argv[r], true);
+        ovsdb_idl_force_reconnect(idl);
+
+        /* Wait for update. */
+        for (;;) {
+            ovsdb_idl_run(idl);
+            ovsdb_idl_check_consistency(idl);
+            if (ovsdb_idl_get_seqno(idl) != seqno) {
+                break;
+            }
+            jsonrpc_run(rpc);
+
+            ovsdb_idl_wait(idl);
+            jsonrpc_wait(rpc);
+            poll_block();
+        }
+
+        seqno = ovsdb_idl_get_seqno(idl);
+
+        bool has_table = idltest_server_has_simple_table(idl);
+        printf("%s remote %s table simple\n",
+               ctx->argv[r], has_table ? "has" : "doesn't have");
+
+        has_table = idltest_server_has_link1_table(idl);
+        printf("%s remote %s table link1\n",
+               ctx->argv[r], has_table ? "has" : "doesn't have");
+
+        has_table = idltest_server_has_link2_table(idl);
+        printf("%s remote %s table link2\n",
+               ctx->argv[r], has_table ? "has" : "doesn't have");
+
+        has_table = idltest_server_has_simple5_table(idl);
+        printf("%s remote %s table simple5\n",
+               ctx->argv[r], has_table ? "has" : "doesn't have");
+
+        bool has_col = idltest_server_has_simple5_table_col_irefmap(idl);
+        printf("%s remote %s col irefmap in table simple5\n",
+               ctx->argv[r], has_col ? "has" : "doesn't have");
+
+        has_col = idltest_server_has_link1_table_col_l2(idl);
+        printf("%s remote %s col l2 in table link1\n",
+               ctx->argv[r], has_col ? "has" : "doesn't have");
+
+        has_col = idltest_server_has_link1_table_col_i(idl);
+        printf("%s remote %s col i in table link1\n",
+               ctx->argv[r], has_col ? "has" : "doesn't have");
+
+        has_col = idltest_server_has_simple7_table_col_id(idl);
+        printf("%s remote %s col id in table simple7\n",
+               ctx->argv[r], has_col ? "has" : "doesn't have");
+
+        printf("--- remote %s done ---\n", ctx->argv[r]);
+    }
+
+    jsonrpc_close(rpc);
+    ovsdb_idl_destroy(idl);
+}
+
 static struct ovs_cmdl_command all_commands[] = {
     { "log-io", NULL, 2, INT_MAX, do_log_io, OVS_RO },
     { "default-atoms", NULL, 0, 0, do_default_atoms, OVS_RO },
@@ -3306,6 +3386,8 @@ static struct ovs_cmdl_command all_commands[] = {
         do_idl_partial_update_map_column, OVS_RO },
     { "idl-partial-update-set-column", NULL, 1, INT_MAX,
         do_idl_partial_update_set_column, OVS_RO },
+    { "idl-table-column-check", NULL, 2, 2,
+        do_idl_table_column_check, OVS_RO },
     { "help", NULL, 0, INT_MAX, do_help, OVS_RO },
     { NULL, NULL, 0, 0, NULL, OVS_RO },
 };