mir/ovs
2
0
Fork 0
mirror of https://github.com/openvswitch/ovs synced 2025-08-31 14:25:26 +00:00
Code Issues Releases Wiki Activity

Add multi-column index support for the Python IDL

Browse Source 
This adds multi-column index support for the Python IDL that is
similar to the feature in the C IDL. Since it adds sortedcontainers
as a dependency and some distros don't yet package it, the library
is copied in-tree and used if sortedcontainers is not installed.

Signed-off-by: Terry Wilson <twilson@redhat.com>
Signed-off-by: Ben Pfaff <blp@ovn.org>
This commit is contained in:
Terry Wilson
2018-04-12 19:24:27 -05:00
committed by Ben Pfaff
parent 112b633627
commit 13973bc415
11 changed files with 3851 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/automake.mk
View File

@@ -10,9 +10,15 @@ ovstest_pyfiles = \
ovs_pyfiles = \
python/ovs/__init__.py \
python/ovs/compat/__init__.py \
python/ovs/compat/sortedcontainers/__init__.py \
python/ovs/compat/sortedcontainers/sortedlist.py \
python/ovs/compat/sortedcontainers/sorteddict.py \
python/ovs/compat/sortedcontainers/sortedset.py \
python/ovs/daemon.py \
python/ovs/fcntl_win.py \
python/ovs/db/__init__.py \
python/ovs/db/custom_index.py \
python/ovs/db/data.py \
python/ovs/db/error.py \
python/ovs/db/idl.py \
@@ -36,7 +42,6 @@ ovs_pyfiles = \
python/ovs/version.py \
python/ovs/vlog.py \
python/ovs/winutils.py
# These python files are used at build time but not runtime,
# so they are not installed.
EXTRA_DIST += \
@@ -46,6 +51,7 @@ EXTRA_DIST += \
# PyPI support.
EXTRA_DIST += \
python/ovs/compat/sortedcontainers/LICENSE \
python/README.rst \
python/setup.py
@@ -57,7 +63,7 @@ EXTRA_DIST += $(PYFILES)
PYCOV_CLEAN_FILES += $(PYFILES:.py=.py,cover)
FLAKE8_PYFILES += \
$(filter-out python/ovs/dirs.py,$(PYFILES)) \
$(filter-out python/ovs/compat/% python/ovs/dirs.py,$(PYFILES)) \
python/setup.py \
python/build/__init__.py \
python/build/nroff.py \

0
python/ovs/compat/__init__.py Normal file
View File

13
python/ovs/compat/sortedcontainers/LICENSE Normal file
View File

@@ -0,0 +1,13 @@
Copyright 2014-2016 Grant Jenks
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

52
python/ovs/compat/sortedcontainers/__init__.py Normal file
View File

@@ -0,0 +1,52 @@
"""Sorted Container Types: SortedList, SortedDict, SortedSet
SortedContainers is an Apache2 licensed containers library, written in
pure-Python, and fast as C-extensions.
Python's standard library is great until you need a sorted collections
type. Many will attest that you can get really far without one, but the moment
you **really need** a sorted list, dict, or set, you're faced with a dozen
different implementations, most using C-extensions without great documentation
and benchmarking.
In Python, we can do better. And we can do it in pure-Python!
::
>>> from sortedcontainers import SortedList, SortedDict, SortedSet
>>> sl = SortedList(xrange(10000000))
>>> 1234567 in sl
True
>>> sl[7654321]
7654321
>>> sl.add(1234567)
>>> sl.count(1234567)
2
>>> sl *= 3
>>> len(sl)
30000003
SortedContainers takes all of the work out of Python sorted types - making your
deployment and use of Python easy. There's no need to install a C compiler or
pre-build and distribute custom extensions. Performance is a feature and
testing has 100% coverage with unit tests and hours of stress.
:copyright: (c) 2016 by Grant Jenks.
:license: Apache 2.0, see LICENSE for more details.
"""
from .sortedlist import SortedList, SortedListWithKey
from .sortedset import SortedSet
from .sorteddict import SortedDict
__all__ = ['SortedList', 'SortedSet', 'SortedDict', 'SortedListWithKey']
__title__ = 'sortedcontainers'
__version__ = '1.5.9'
__build__ = 0x010509
__author__ = 'Grant Jenks'
__license__ = 'Apache 2.0'
__copyright__ = 'Copyright 2016 Grant Jenks'

741
python/ovs/compat/sortedcontainers/sorteddict.py Normal file
View File

@@ -0,0 +1,741 @@
"""Sorted dictionary implementation.
"""
from collections import Set, Sequence
from collections import KeysView as AbstractKeysView
from collections import ValuesView as AbstractValuesView
from collections import ItemsView as AbstractItemsView
from sys import hexversion
from .sortedlist import SortedList, recursive_repr, SortedListWithKey
from .sortedset import SortedSet
NONE = object()
class _IlocWrapper(object):
"Positional indexing support for sorted dictionary objects."
# pylint: disable=protected-access, too-few-public-methods
def __init__(self, _dict):
self._dict = _dict
def __len__(self):
return len(self._dict)
def __getitem__(self, index):
"""
Very efficiently return the key at index *index* in iteration. Supports
negative indices and slice notation. Raises IndexError on invalid
*index*.
"""
return self._dict._list[index]
def __delitem__(self, index):
"""
Remove the ``sdict[sdict.iloc[index]]`` from *sdict*. Supports negative
indices and slice notation. Raises IndexError on invalid *index*.
"""
_dict = self._dict
_list = _dict._list
_delitem = _dict._delitem
if isinstance(index, slice):
keys = _list[index]
del _list[index]
for key in keys:
_delitem(key)
else:
key = _list[index]
del _list[index]
_delitem(key)
class SortedDict(dict):
"""SortedDict provides the same methods as a dict. Additionally, SortedDict
efficiently maintains its keys in sorted order. Consequently, the keys
method will return the keys in sorted order, the popitem method will remove
the item with the highest key, etc.
"""
def __init__(self, *args, **kwargs):
"""SortedDict provides the same methods as a dict. Additionally, SortedDict
efficiently maintains its keys in sorted order. Consequently, the keys
method will return the keys in sorted order, the popitem method will
remove the item with the highest key, etc.
An optional *key* argument defines a callable that, like the `key`
argument to Python's `sorted` function, extracts a comparison key from
each dict key. If no function is specified, the default compares the
dict keys directly. The `key` argument must be provided as a positional
argument and must come before all other arguments.
An optional *iterable* argument provides an initial series of items to
populate the SortedDict. Each item in the series must itself contain
two items. The first is used as a key in the new dictionary, and the
second as the key's value. If a given key is seen more than once, the
last value associated with it is retained in the new dictionary.
If keyword arguments are given, the keywords themselves with their
associated values are added as items to the dictionary. If a key is
specified both in the positional argument and as a keyword argument, the
value associated with the keyword is retained in the dictionary. For
example, these all return a dictionary equal to ``{"one": 2, "two":
3}``:
* ``SortedDict(one=2, two=3)``
* ``SortedDict({'one': 2, 'two': 3})``
* ``SortedDict(zip(('one', 'two'), (2, 3)))``
* ``SortedDict([['two', 3], ['one', 2]])``
The first example only works for keys that are valid Python
identifiers; the others work with any valid keys.
"""
# pylint: disable=super-init-not-called
if args and (args[0] is None or callable(args[0])):
self._key = args[0]
args = args[1:]
else:
self._key = None
if self._key is None:
self._list = SortedList()
else:
self._list = SortedListWithKey(key=self._key)
# Cache function pointers to dict methods.
_dict = super(SortedDict, self)
self._dict = _dict
self._clear = _dict.clear
self._delitem = _dict.__delitem__
self._iter = _dict.__iter__
self._pop = _dict.pop
self._setdefault = _dict.setdefault
self._setitem = _dict.__setitem__
self._dict_update = _dict.update
# Cache function pointers to SortedList methods.
_list = self._list
self._list_add = _list.add
self.bisect_left = _list.bisect_left
self.bisect = _list.bisect_right
self.bisect_right = _list.bisect_right
self._list_clear = _list.clear
self.index = _list.index
self._list_pop = _list.pop
self._list_remove = _list.remove
self._list_update = _list.update
self.irange = _list.irange
self.islice = _list.islice
self._reset = _list._reset # pylint: disable=protected-access
if self._key is not None:
self.bisect_key_left = _list.bisect_key_left
self.bisect_key_right = _list.bisect_key_right
self.bisect_key = _list.bisect_key
self.irange_key = _list.irange_key
self.iloc = _IlocWrapper(self)
self._update(*args, **kwargs)
@property
def key(self):
"""Key function used to extract comparison key for sorting."""
return self._key
def clear(self):
"""Remove all elements from the dictionary."""
self._clear()
self._list_clear()
def __delitem__(self, key):
"""
Remove ``d[key]`` from *d*. Raises a KeyError if *key* is not in the
dictionary.
"""
self._delitem(key)
self._list_remove(key)
def __iter__(self):
"""
Return an iterator over the sorted keys of the dictionary.
Iterating the Mapping while adding or deleting keys may raise a
`RuntimeError` or fail to iterate over all entries.
"""
return iter(self._list)
def __reversed__(self):
"""
Return a reversed iterator over the sorted keys of the dictionary.
Iterating the Mapping while adding or deleting keys may raise a
`RuntimeError` or fail to iterate over all entries.
"""
return reversed(self._list)
def __setitem__(self, key, value):
"""Set `d[key]` to *value*."""
if key not in self:
self._list_add(key)
self._setitem(key, value)
def copy(self):
"""Return a shallow copy of the sorted dictionary."""
return self.__class__(self._key, self._iteritems())
__copy__ = copy
@classmethod
def fromkeys(cls, seq, value=None):
"""
Create a new dictionary with keys from *seq* and values set to *value*.
"""
return cls((key, value) for key in seq)
if hexversion < 0x03000000:
def items(self):
"""
Return a list of the dictionary's items (``(key, value)`` pairs).
"""
return list(self._iteritems())
else:
def items(self):
"""
Return a new ItemsView of the dictionary's items. In addition to
the methods provided by the built-in `view` the ItemsView is
indexable (e.g. ``d.items()[5]``).
"""
return ItemsView(self)
def iteritems(self):
"""
Return an iterator over the items (``(key, value)`` pairs).
Iterating the Mapping while adding or deleting keys may raise a
`RuntimeError` or fail to iterate over all entries.
"""
return iter((key, self[key]) for key in self._list)
_iteritems = iteritems
if hexversion < 0x03000000:
def keys(self):
"""Return a SortedSet of the dictionary's keys."""
return SortedSet(self._list, key=self._key)
else:
def keys(self):
"""
Return a new KeysView of the dictionary's keys. In addition to the
methods provided by the built-in `view` the KeysView is indexable
(e.g. ``d.keys()[5]``).
"""
return KeysView(self)
def iterkeys(self):
"""
Return an iterator over the sorted keys of the Mapping.
Iterating the Mapping while adding or deleting keys may raise a
`RuntimeError` or fail to iterate over all entries.
"""
return iter(self._list)
if hexversion < 0x03000000:
def values(self):
"""Return a list of the dictionary's values."""
return list(self._itervalues())
else:
def values(self):
"""
Return a new :class:`ValuesView` of the dictionary's values.
In addition to the methods provided by the built-in `view` the
ValuesView is indexable (e.g., ``d.values()[5]``).
"""
return ValuesView(self)
def itervalues(self):
"""
Return an iterator over the values of the Mapping.
Iterating the Mapping while adding or deleting keys may raise a
`RuntimeError` or fail to iterate over all entries.
"""
return iter(self[key] for key in self._list)
_itervalues = itervalues
def pop(self, key, default=NONE):
"""
If *key* is in the dictionary, remove it and return its value,
else return *default*. If *default* is not given and *key* is not in
the dictionary, a KeyError is raised.
"""
if key in self:
self._list_remove(key)
return self._pop(key)
else:
if default is NONE:
raise KeyError(key)
else:
return default
def popitem(self, last=True):
"""
Remove and return a ``(key, value)`` pair from the dictionary. If
last=True (default) then remove the *greatest* `key` from the
diciontary. Else, remove the *least* key from the dictionary.
If the dictionary is empty, calling `popitem` raises a
KeyError`.
"""
if not self:
raise KeyError('popitem(): dictionary is empty')
key = self._list_pop(-1 if last else 0)
value = self._pop(key)
return (key, value)
def peekitem(self, index=-1):
"""Return (key, value) item pair at index.
Unlike ``popitem``, the sorted dictionary is not modified. Index
defaults to -1, the last/greatest key in the dictionary. Specify
``index=0`` to lookup the first/least key in the dictiony.
If index is out of range, raise IndexError.
"""
key = self._list[index]
return key, self[key]
def setdefault(self, key, default=None):
"""
If *key* is in the dictionary, return its value. If not, insert *key*
with a value of *default* and return *default*. *default* defaults to
``None``.
"""
if key in self:
return self[key]
self._setitem(key, default)
self._list_add(key)
return default
def update(self, *args, **kwargs):
"""
Update the dictionary with the key/value pairs from *other*, overwriting
existing keys.
*update* accepts either another dictionary object or an iterable of
key/value pairs (as a tuple or other iterable of length two). If
keyword arguments are specified, the dictionary is then updated with
those key/value pairs: ``d.update(red=1, blue=2)``.
"""
if not self:
self._dict_update(*args, **kwargs)
self._list_update(self._iter())
return
if not kwargs and len(args) == 1 and isinstance(args[0], dict):
pairs = args[0]
else:
pairs = dict(*args, **kwargs)
if (10 * len(pairs)) > len(self):
self._dict_update(pairs)
self._list_clear()
self._list_update(self._iter())
else:
for key in pairs:
self[key] = pairs[key]
_update = update
if hexversion >= 0x02070000:
def viewkeys(self):
"Return ``KeysView`` of dictionary keys."
return KeysView(self)
def viewvalues(self):
"Return ``ValuesView`` of dictionary values."
return ValuesView(self)
def viewitems(self):
"Return ``ItemsView`` of dictionary (key, value) item pairs."
return ItemsView(self)
def __reduce__(self):
return (self.__class__, (self._key, list(self._iteritems())))
@recursive_repr
def __repr__(self):
_key = self._key
name = type(self).__name__
key = '' if _key is None else '{0!r}, '.format(_key)
func = '{0!r}: {1!r}'.format
items = ', '.join(func(key, self[key]) for key in self._list)
return '{0}({1}{{{2}}})'.format(name, key, items)
def _check(self):
# pylint: disable=protected-access
self._list._check()
assert len(self) == len(self._list)
assert all(key in self for key in self._list)
class KeysView(AbstractKeysView, Set, Sequence):
"""
A KeysView object is a dynamic view of the dictionary's keys, which
means that when the dictionary's keys change, the view reflects
those changes.
The KeysView class implements the Set and Sequence Abstract Base Classes.
"""
# pylint: disable=too-many-ancestors
if hexversion < 0x03000000:
def __init__(self, sorted_dict):
"""
Initialize a KeysView from a SortedDict container as *sorted_dict*.
"""
# pylint: disable=super-init-not-called, protected-access
self._list = sorted_dict._list
self._view = sorted_dict._dict.viewkeys()
else:
def __init__(self, sorted_dict):
"""
Initialize a KeysView from a SortedDict container as *sorted_dict*.
"""
# pylint: disable=super-init-not-called, protected-access
self._list = sorted_dict._list
self._view = sorted_dict._dict.keys()
def __len__(self):
"""Return the number of entries in the dictionary."""
return len(self._view)
def __contains__(self, key):
"""
Return True if and only if *key* is one of the underlying dictionary's
keys.
"""
return key in self._view
def __iter__(self):
"""
Return an iterable over the keys in the dictionary. Keys are iterated
over in their sorted order.
Iterating views while adding or deleting entries in the dictionary may
raise a `RuntimeError` or fail to iterate over all entries.
"""
return iter(self._list)
def __getitem__(self, index):
"""Return the key at position *index*."""
return self._list[index]
def __reversed__(self):
"""
Return a reversed iterable over the keys in the dictionary. Keys are
iterated over in their reverse sort order.
Iterating views while adding or deleting entries in the dictionary may
raise a RuntimeError or fail to iterate over all entries.
"""
return reversed(self._list)
def index(self, value, start=None, stop=None):
"""
Return the smallest *k* such that `keysview[k] == value` and `start <= k
< end`. Raises `KeyError` if *value* is not present. *stop* defaults
to the end of the set. *start* defaults to the beginning. Negative
indexes are supported, as for slice indices.
"""
# pylint: disable=arguments-differ
return self._list.index(value, start, stop)
def count(self, value):
"""Return the number of occurrences of *value* in the set."""
return 1 if value in self._view else 0
def __eq__(self, that):
"""Test set-like equality with *that*."""
return self._view == that
def __ne__(self, that):
"""Test set-like inequality with *that*."""
return self._view != that
def __lt__(self, that):
"""Test whether self is a proper subset of *that*."""
return self._view < that
def __gt__(self, that):
"""Test whether self is a proper superset of *that*."""
return self._view > that
def __le__(self, that):
"""Test whether self is contained within *that*."""
return self._view <= that
def __ge__(self, that):
"""Test whether *that* is contained within self."""
return self._view >= that
def __and__(self, that):
"""Return a SortedSet of the intersection of self and *that*."""
return SortedSet(self._view & that)
def __or__(self, that):
"""Return a SortedSet of the union of self and *that*."""
return SortedSet(self._view | that)
def __sub__(self, that):
"""Return a SortedSet of the difference of self and *that*."""
return SortedSet(self._view - that)
def __xor__(self, that):
"""Return a SortedSet of the symmetric difference of self and *that*."""
return SortedSet(self._view ^ that)
if hexversion < 0x03000000:
def isdisjoint(self, that):
"""Return True if and only if *that* is disjoint with self."""
# pylint: disable=arguments-differ
return not any(key in self._list for key in that)
else:
def isdisjoint(self, that):
"""Return True if and only if *that* is disjoint with self."""
# pylint: disable=arguments-differ
return self._view.isdisjoint(that)
@recursive_repr
def __repr__(self):
return 'SortedDict_keys({0!r})'.format(list(self))
class ValuesView(AbstractValuesView, Sequence):
"""
A ValuesView object is a dynamic view of the dictionary's values, which
means that when the dictionary's values change, the view reflects those
changes.
The ValuesView class implements the Sequence Abstract Base Class.
"""
# pylint: disable=too-many-ancestors
if hexversion < 0x03000000:
def __init__(self, sorted_dict):
"""
Initialize a ValuesView from a SortedDict container as
*sorted_dict*.
"""
# pylint: disable=super-init-not-called, protected-access
self._dict = sorted_dict
self._list = sorted_dict._list
self._view = sorted_dict._dict.viewvalues()
else:
def __init__(self, sorted_dict):
"""
Initialize a ValuesView from a SortedDict container as
*sorted_dict*.
"""
# pylint: disable=super-init-not-called, protected-access
self._dict = sorted_dict
self._list = sorted_dict._list
self._view = sorted_dict._dict.values()
def __len__(self):
"""Return the number of entries in the dictionary."""
return len(self._dict)
def __contains__(self, value):
"""
Return True if and only if *value* is in the underlying Mapping's
values.
"""
return value in self._view
def __iter__(self):
"""
Return an iterator over the values in the dictionary. Values are
iterated over in sorted order of the keys.
Iterating views while adding or deleting entries in the dictionary may
raise a `RuntimeError` or fail to iterate over all entries.
"""
_dict = self._dict
return iter(_dict[key] for key in self._list)
def __getitem__(self, index):
"""
Efficiently return value at *index* in iteration.
Supports slice notation and negative indexes.
"""
_dict, _list = self._dict, self._list
if isinstance(index, slice):
return [_dict[key] for key in _list[index]]
return _dict[_list[index]]
def __reversed__(self):
"""
Return a reverse iterator over the values in the dictionary. Values are
iterated over in reverse sort order of the keys.
Iterating views while adding or deleting entries in the dictionary may
raise a `RuntimeError` or fail to iterate over all entries.
"""
_dict = self._dict
return iter(_dict[key] for key in reversed(self._list))
def index(self, value):
"""
Return index of *value* in self.
Raises ValueError if *value* is not found.
"""
# pylint: disable=arguments-differ
for idx, val in enumerate(self):
if value == val:
return idx
raise ValueError('{0!r} is not in dict'.format(value))
if hexversion < 0x03000000:
def count(self, value):
"""Return the number of occurrences of *value* in self."""
return sum(1 for val in self._dict.itervalues() if val == value)
else:
def count(self, value):
"""Return the number of occurrences of *value* in self."""
return sum(1 for val in self._dict.values() if val == value)
def __lt__(self, that):
raise TypeError
def __gt__(self, that):
raise TypeError
def __le__(self, that):
raise TypeError
def __ge__(self, that):
raise TypeError
def __and__(self, that):
raise TypeError
def __or__(self, that):
raise TypeError
def __sub__(self, that):
raise TypeError
def __xor__(self, that):
raise TypeError
@recursive_repr
def __repr__(self):
return 'SortedDict_values({0!r})'.format(list(self))
class ItemsView(AbstractItemsView, Set, Sequence):
"""
An ItemsView object is a dynamic view of the dictionary's ``(key,
value)`` pairs, which means that when the dictionary changes, the
view reflects those changes.
The ItemsView class implements the Set and Sequence Abstract Base Classes.
However, the set-like operations (``&``, ``|``, ``-``, ``^``) will only
operate correctly if all of the dictionary's values are hashable.
"""
# pylint: disable=too-many-ancestors
if hexversion < 0x03000000:
def __init__(self, sorted_dict):
"""
Initialize an ItemsView from a SortedDict container as
*sorted_dict*.
"""
# pylint: disable=super-init-not-called, protected-access
self._dict = sorted_dict
self._list = sorted_dict._list
self._view = sorted_dict._dict.viewitems()
else:
def __init__(self, sorted_dict):
"""
Initialize an ItemsView from a SortedDict container as
*sorted_dict*.
"""
# pylint: disable=super-init-not-called, protected-access
self._dict = sorted_dict
self._list = sorted_dict._list
self._view = sorted_dict._dict.items()
def __len__(self):
"""Return the number of entries in the dictionary."""
return len(self._view)
def __contains__(self, key):
"""
Return True if and only if *key* is one of the underlying dictionary's
items.
"""
return key in self._view
def __iter__(self):
"""
Return an iterable over the items in the dictionary. Items are iterated
over in their sorted order.
Iterating views while adding or deleting entries in the dictionary may
raise a `RuntimeError` or fail to iterate over all entries.
"""
_dict = self._dict
return iter((key, _dict[key]) for key in self._list)
def __getitem__(self, index):
"""Return the item as position *index*."""
_dict, _list = self._dict, self._list
if isinstance(index, slice):
return [(key, _dict[key]) for key in _list[index]]
key = _list[index]
return (key, _dict[key])
def __reversed__(self):
"""
Return a reversed iterable over the items in the dictionary. Items are
iterated over in their reverse sort order.
Iterating views while adding or deleting entries in the dictionary may
raise a RuntimeError or fail to iterate over all entries.
"""
_dict = self._dict
return iter((key, _dict[key]) for key in reversed(self._list))
def index(self, key, start=None, stop=None):
"""
Return the smallest *k* such that `itemssview[k] == key` and `start <= k
< end`. Raises `KeyError` if *key* is not present. *stop* defaults
to the end of the set. *start* defaults to the beginning. Negative
indexes are supported, as for slice indices.
"""
# pylint: disable=arguments-differ
temp, value = key
pos = self._list.index(temp, start, stop)
if value == self._dict[temp]:
return pos
else:
raise ValueError('{0!r} is not in dict'.format(key))
def count(self, item):
"""Return the number of occurrences of *item* in the set."""
# pylint: disable=arguments-differ
key, value = item
return 1 if key in self._dict and self._dict[key] == value else 0
def __eq__(self, that):
"""Test set-like equality with *that*."""
return self._view == that
def __ne__(self, that):
"""Test set-like inequality with *that*."""
return self._view != that
def __lt__(self, that):
"""Test whether self is a proper subset of *that*."""
return self._view < that
def __gt__(self, that):
"""Test whether self is a proper superset of *that*."""
return self._view > that
def __le__(self, that):
"""Test whether self is contained within *that*."""
return self._view <= that
def __ge__(self, that):
"""Test whether *that* is contained within self."""
return self._view >= that
def __and__(self, that):
"""Return a SortedSet of the intersection of self and *that*."""
return SortedSet(self._view & that)
def __or__(self, that):
"""Return a SortedSet of the union of self and *that*."""
return SortedSet(self._view | that)
def __sub__(self, that):
"""Return a SortedSet of the difference of self and *that*."""
return SortedSet(self._view - that)
def __xor__(self, that):
"""Return a SortedSet of the symmetric difference of self and *that*."""
return SortedSet(self._view ^ that)
if hexversion < 0x03000000:
def isdisjoint(self, that):
"""Return True if and only if *that* is disjoint with self."""
# pylint: disable=arguments-differ
_dict = self._dict
for key, value in that:
if key in _dict and _dict[key] == value:
return False
return True
else:
def isdisjoint(self, that):
"""Return True if and only if *that* is disjoint with self."""
# pylint: disable=arguments-differ
return self._view.isdisjoint(that)
@recursive_repr
def __repr__(self):
return 'SortedDict_items({0!r})'.format(list(self))

2508
python/ovs/compat/sortedcontainers/sortedlist.py Normal file
View File

File diff suppressed because it is too large Load Diff

327
python/ovs/compat/sortedcontainers/sortedset.py Normal file
View File

@@ -0,0 +1,327 @@
"""Sorted set implementation.
"""
from collections import Set, MutableSet, Sequence
from itertools import chain
import operator as op
from .sortedlist import SortedList, recursive_repr, SortedListWithKey
class SortedSet(MutableSet, Sequence):
"""
A `SortedSet` provides the same methods as a `set`. Additionally, a
`SortedSet` maintains its items in sorted order, allowing the `SortedSet` to
be indexed.
Unlike a `set`, a `SortedSet` requires items be hashable and comparable.
"""
# pylint: disable=too-many-ancestors
def __init__(self, iterable=None, key=None):
"""
A `SortedSet` provides the same methods as a `set`. Additionally, a
`SortedSet` maintains its items in sorted order, allowing the
`SortedSet` to be indexed.
An optional *iterable* provides an initial series of items to populate
the `SortedSet`.
An optional *key* argument defines a callable that, like the `key`
argument to Python's `sorted` function, extracts a comparison key from
each set item. If no function is specified, the default compares the
set items directly.
"""
self._key = key
if not hasattr(self, '_set'):
self._set = set()
_set = self._set
self.isdisjoint = _set.isdisjoint
self.issubset = _set.issubset
self.issuperset = _set.issuperset
if key is None:
self._list = SortedList(self._set)
else:
self._list = SortedListWithKey(self._set, key=key)
_list = self._list
self.bisect_left = _list.bisect_left
self.bisect = _list.bisect
self.bisect_right = _list.bisect_right
self.index = _list.index
self.irange = _list.irange
self.islice = _list.islice
self._reset = _list._reset # pylint: disable=protected-access
if key is not None:
self.bisect_key_left = _list.bisect_key_left
self.bisect_key_right = _list.bisect_key_right
self.bisect_key = _list.bisect_key
self.irange_key = _list.irange_key
if iterable is not None:
self._update(iterable)
@property
def key(self):
"""Key function used to extract comparison key for sorting."""
return self._key
@classmethod
def _fromset(cls, values, key=None):
"""Initialize sorted set from existing set."""
sorted_set = object.__new__(cls)
sorted_set._set = values # pylint: disable=protected-access
sorted_set.__init__(key=key)
return sorted_set
def __contains__(self, value):
"""Return True if and only if *value* is an element in the set."""
return value in self._set
def __getitem__(self, index):
"""
Return the element at position *index*.
Supports slice notation and negative indexes.
"""
return self._list[index]
def __delitem__(self, index):
"""
Remove the element at position *index*.
Supports slice notation and negative indexes.
"""
_set = self._set
_list = self._list
if isinstance(index, slice):
values = _list[index]
_set.difference_update(values)
else:
value = _list[index]
_set.remove(value)
del _list[index]
def _make_cmp(self, set_op, doc):
"Make comparator method."
def comparer(self, that):
"Compare method for sorted set and set-like object."
# pylint: disable=protected-access
if isinstance(that, SortedSet):
return set_op(self._set, that._set)
elif isinstance(that, Set):
return set_op(self._set, that)
return NotImplemented
comparer.__name__ = '__{0}__'.format(set_op.__name__)
doc_str = 'Return True if and only if Set is {0} `that`.'
comparer.__doc__ = doc_str.format(doc)
return comparer
__eq__ = _make_cmp(None, op.eq, 'equal to')
__ne__ = _make_cmp(None, op.ne, 'not equal to')
__lt__ = _make_cmp(None, op.lt, 'a proper subset of')
__gt__ = _make_cmp(None, op.gt, 'a proper superset of')
__le__ = _make_cmp(None, op.le, 'a subset of')
__ge__ = _make_cmp(None, op.ge, 'a superset of')
def __len__(self):
"""Return the number of elements in the set."""
return len(self._set)
def __iter__(self):
"""
Return an iterator over the Set. Elements are iterated in their sorted
order.
Iterating the Set while adding or deleting values may raise a
`RuntimeError` or fail to iterate over all entries.
"""
return iter(self._list)
def __reversed__(self):
"""
Return an iterator over the Set. Elements are iterated in their reverse
sorted order.
Iterating the Set while adding or deleting values may raise a
`RuntimeError` or fail to iterate over all entries.
"""
return reversed(self._list)
def add(self, value):
"""Add the element *value* to the set."""
_set = self._set
if value not in _set:
_set.add(value)
self._list.add(value)
def clear(self):
"""Remove all elements from the set."""
self._set.clear()
self._list.clear()
def copy(self):
"""Create a shallow copy of the sorted set."""
return self._fromset(set(self._set), key=self._key)
__copy__ = copy
def count(self, value):
"""Return the number of occurrences of *value* in the set."""
return 1 if value in self._set else 0
def discard(self, value):
"""
Remove the first occurrence of *value*. If *value* is not a member,
does nothing.
"""
_set = self._set
if value in _set:
_set.remove(value)
self._list.discard(value)
def pop(self, index=-1):
"""
Remove and return item at *index* (default last). Raises IndexError if
set is empty or index is out of range. Negative indexes are supported,
as for slice indices.
"""
# pylint: disable=arguments-differ
value = self._list.pop(index)
self._set.remove(value)
return value
def remove(self, value):
"""
Remove first occurrence of *value*. Raises ValueError if
*value* is not present.
"""
self._set.remove(value)
self._list.remove(value)
def difference(self, *iterables):
"""
Return a new set with elements in the set that are not in the
*iterables*.
"""
diff = self._set.difference(*iterables)
return self._fromset(diff, key=self._key)
__sub__ = difference
__rsub__ = __sub__
def difference_update(self, *iterables):
"""
Update the set, removing elements found in keeping only elements
found in any of the *iterables*.
"""
_set = self._set
values = set(chain(*iterables))
if (4 * len(values)) > len(_set):
_list = self._list
_set.difference_update(values)
_list.clear()
_list.update(_set)
else:
_discard = self.discard
for value in values:
_discard(value)
return self
__isub__ = difference_update
def intersection(self, *iterables):
"""
Return a new set with elements common to the set and all *iterables*.
"""
comb = self._set.intersection(*iterables)
return self._fromset(comb, key=self._key)
__and__ = intersection
__rand__ = __and__
def intersection_update(self, *iterables):
"""
Update the set, keeping only elements found in it and all *iterables*.
"""
_set = self._set
_list = self._list
_set.intersection_update(*iterables)
_list.clear()
_list.update(_set)
return self
__iand__ = intersection_update
def symmetric_difference(self, that):
"""
Return a new set with elements in either *self* or *that* but not both.
"""
diff = self._set.symmetric_difference(that)
return self._fromset(diff, key=self._key)
__xor__ = symmetric_difference
__rxor__ = __xor__
def symmetric_difference_update(self, that):
"""
Update the set, keeping only elements found in either *self* or *that*,
but not in both.
"""
_set = self._set
_list = self._list
_set.symmetric_difference_update(that)
_list.clear()
_list.update(_set)
return self
__ixor__ = symmetric_difference_update
def union(self, *iterables):
"""
Return a new SortedSet with elements from the set and all *iterables*.
"""
return self.__class__(chain(iter(self), *iterables), key=self._key)
__or__ = union
__ror__ = __or__
def update(self, *iterables):
"""Update the set, adding elements from all *iterables*."""
_set = self._set
values = set(chain(*iterables))
if (4 * len(values)) > len(_set):
_list = self._list
_set.update(values)
_list.clear()
_list.update(_set)
else:
_add = self.add
for value in values:
_add(value)
return self
__ior__ = update
_update = update
def __reduce__(self):
return (type(self), (self._set, self._key))
@recursive_repr
def __repr__(self):
_key = self._key
key = '' if _key is None else ', key={0!r}'.format(_key)
name = type(self).__name__
return '{0}({1!r}{2})'.format(name, list(self), key)
def _check(self):
# pylint: disable=protected-access
self._list._check()
assert len(self._set) == len(self._list)
_set = self._set
assert all(val in _set for val in self._list)

154
python/ovs/db/custom_index.py Normal file
View File

@@ -0,0 +1,154 @@
import collections
import functools
import operator
try:
from UserDict import IterableUserDict as DictBase
except ImportError:
from collections import UserDict as DictBase
try:
import sortedcontainers
except ImportError:
from ovs.compat import sortedcontainers
from ovs.db import data
OVSDB_INDEX_ASC = "ASC"
OVSDB_INDEX_DESC = "DESC"
ColumnIndex = collections.namedtuple('ColumnIndex',
['column', 'direction', 'key'])
class MultiColumnIndex(object):
def __init__(self, name):
self.name = name
self.columns = []
self.clear()
def __repr__(self):
return "{}(name={})".format(self.__class__.__name__, self.name)
def __str__(self):
return repr(self) + " columns={} values={}".format(
self.columns, [str(v) for v in self.values])
def add_column(self, column, direction=OVSDB_INDEX_ASC, key=None):
self.columns.append(ColumnIndex(column, direction,
key or operator.attrgetter(column)))
def add_columns(self, *columns):
self.columns.extend(ColumnIndex(col, OVSDB_INDEX_ASC,
operator.attrgetter(col))
for col in columns)
def _cmp(self, a, b):
for col, direction, key in self.columns:
aval, bval = key(a), key(b)
if aval == bval:
continue
result = (aval > bval) - (aval < bval)
return result if direction == OVSDB_INDEX_ASC else -result
return 0
def index_entry_from_row(self, row):
return row._table.rows.IndexEntry(
uuid=row.uuid,
**{c.column: getattr(row, c.column) for c in self.columns})
def add(self, row):
if not all(hasattr(row, col.column) for col in self.columns):
# This is a new row, but it hasn't had the necessary columns set
# We'll add it later
return
self.values.add(self.index_entry_from_row(row))
def remove(self, row):
self.values.remove(self.index_entry_from_row(row))
def clear(self):
self.values = sortedcontainers.SortedListWithKey(
key=functools.cmp_to_key(self._cmp))
def irange(self, start, end):
return iter(r._table.rows[r.uuid]
for r in self.values.irange(start, end))
def __iter__(self):
return iter(r._table.rows[r.uuid] for r in self.values)
class IndexedRows(DictBase, object):
def __init__(self, table, *args, **kwargs):
super(IndexedRows, self).__init__(*args, **kwargs)
self.table = table
self.indexes = {}
self.IndexEntry = IndexEntryClass(table)
def index_create(self, name):
if name in self.indexes:
raise ValueError("An index named {} already exists".format(name))
index = self.indexes[name] = MultiColumnIndex(name)
return index
def __setitem__(self, key, item):
self.data[key] = item
for index in self.indexes.values():
index.add(item)
def __delitem__(self, key):
val = self.data[key]
del self.data[key]
for index in self.indexes.values():
index.remove(val)
def clear(self):
self.data.clear()
for index in self.indexes.values():
index.clear()
# Nothing uses the methods below, though they'd be easy to implement
def update(self, dict=None, **kwargs):
raise NotImplementedError()
def setdefault(self, key, failobj=None):
raise NotImplementedError()
def pop(self, key, *args):
raise NotImplementedError()
def popitem(self):
raise NotImplementedError()
@classmethod
def fromkeys(cls, iterable, value=None):
raise NotImplementedError()
def IndexEntryClass(table):
"""Create a class used represent Rows in indexes
ovs.db.idl.Row, being inherently tied to transaction processing and being
initialized with dicts of Datums, is not really useable as an object to
pass to and store in indexes. This method will create a class named after
the table's name that is initialized with that Table Row's default values.
For example:
Port = IndexEntryClass(idl.tables['Port'])
will create a Port class. This class can then be used to search custom
indexes. For example:
for port in idx.iranage(Port(name="test1"), Port(name="test9")):
...
"""
def defaults_uuid_to_row(atom, base):
return atom.value
columns = ['uuid'] + list(table.columns.keys())
cls = collections.namedtuple(table.name, columns)
cls._table = table
cls.__new__.__defaults__ = (None,) + tuple(
data.Datum.default(c.type).to_python(defaults_uuid_to_row)
for c in table.columns.values())
return cls

55
python/ovs/db/idl.py
View File

@@ -22,6 +22,7 @@ import ovs.jsonrpc
import ovs.ovsuuid
import ovs.poller
import ovs.vlog
from ovs.db import custom_index
from ovs.db import error
import six
@@ -148,11 +149,23 @@ class Idl(object):
if not hasattr(column, 'alert'):
column.alert = True
table.need_table = False
table.rows = {}
table.rows = custom_index.IndexedRows(table)
table.idl = self
table.condition = [True]
table.cond_changed = False
def index_create(self, table, name):
"""Create a named multi-column index on a table"""
return self.tables[table].rows.index_create(name)
def index_irange(self, table, name, start, end):
"""Return items in a named index between start/end inclusive"""
return self.tables[table].rows.indexes[name].irange(start, end)
def index_equal(self, table, name, value):
"""Return items in a named index matching a value"""
return self.tables[table].rows.indexes[name].irange(value, value)
def close(self):
"""Closes the connection to the database. The IDL will no longer
update."""
@@ -359,7 +372,7 @@ class Idl(object):
for table in six.itervalues(self.tables):
if table.rows:
changed = True
table.rows = {}
table.rows = custom_index.IndexedRows(table)
if changed:
self.change_seqno += 1
@@ -512,8 +525,9 @@ class Idl(object):
else:
row_update = row_update['initial']
self.__add_default(table, row_update)
if self.__row_update(table, row, row_update):
changed = True
changed = self.__row_update(table, row, row_update)
table.rows[uuid] = row
if changed:
self.notify(ROW_CREATE, row)
elif "modify" in row_update:
if not row:
@@ -543,15 +557,19 @@ class Idl(object):
% (uuid, table.name))
elif not old:
# Insert row.
op = ROW_CREATE
if not row:
row = self.__create_row(table, uuid)
changed = True
else:
# XXX rate-limit
op = ROW_UPDATE
vlog.warn("cannot add existing row %s to table %s"
% (uuid, table.name))
if self.__row_update(table, row, new):
changed = True
changed |= self.__row_update(table, row, new)
if op == ROW_CREATE:
table.rows[uuid] = row
if changed:
self.notify(ROW_CREATE, row)
else:
op = ROW_UPDATE
@@ -562,8 +580,10 @@ class Idl(object):
# XXX rate-limit
vlog.warn("cannot modify missing row %s in table %s"
% (uuid, table.name))
if self.__row_update(table, row, new):
changed = True
changed |= self.__row_update(table, row, new)
if op == ROW_CREATE:
table.rows[uuid] = row
if changed:
self.notify(op, row, Row.from_json(self, table, uuid, old))
return changed
@@ -639,8 +659,7 @@ class Idl(object):
data = {}
for column in six.itervalues(table.columns):
data[column.name] = ovs.db.data.Datum.default(column.type)
row = table.rows[uuid] = Row(self, table, uuid, data)
return row
return Row(self, table, uuid, data)
def __error(self):
self._session.force_reconnect()
@@ -849,7 +868,17 @@ class Row(object):
vlog.err("attempting to write bad value to column %s (%s)"
% (column_name, e))
return
# Remove prior version of the Row from the index if it has the indexed
# column set, and the column changing is an indexed column
if hasattr(self, column_name):
for idx in self._table.rows.indexes.values():
if column_name in (c.column for c in idx.columns):
idx.remove(self)
self._idl.txn._write(self, column, datum)
for idx in self._table.rows.indexes.values():
# Only update the index if indexed columns change
if column_name in (c.column for c in idx.columns):
idx.add(self)
def addvalue(self, column_name, key):
self._idl.txn._txn_rows[self.uuid] = self
@@ -977,8 +1006,8 @@ class Row(object):
del self._idl.txn._txn_rows[self.uuid]
else:
self._idl.txn._txn_rows[self.uuid] = self
self.__dict__["_changes"] = None
del self._table.rows[self.uuid]
self.__dict__["_changes"] = None
def fetch(self, column_name):
self._idl.txn._fetch(self, column_name)
@@ -1150,6 +1179,10 @@ class Transaction(object):
for row in six.itervalues(self._txn_rows):
if row._changes is None:
# If we add the deleted row back to rows with _changes == None
# then __getattr__ will not work for the indexes
row.__dict__["_changes"] = {}
row.__dict__["_mutations"] = {}
row._table.rows[row.uuid] = row
elif row._data is None:
del row._table.rows[row.uuid]

1
python/setup.py
View File

@@ -81,6 +81,7 @@ setup_args = dict(
ext_modules=[setuptools.Extension("ovs._json", sources=["ovs/_json.c"],
libraries=['openvswitch'])],
cmdclass={'build_ext': try_build_ext},
install_requires=['sortedcontainers'],
)
try:

7
tests/test-ovsdb.py
View File

@@ -289,10 +289,7 @@ def idltest_find_simple2(idl, i):
def idltest_find_simple3(idl, i):
for row in six.itervalues(idl.tables["simple3"].rows):
if row.name == i:
return row
return None
return next(idl.index_equal("simple3", "simple3_by_name", i), None)
def idl_set(idl, commands, step):
@@ -579,6 +576,8 @@ def do_idl(schema_file, remote, *commands):
else:
schema_helper.register_all()
idl = ovs.db.idl.Idl(remote, schema_helper)
if "simple3" in idl.tables:
idl.index_create("simple3", "simple3_by_name")
if commands:
error, stream = ovs.stream.Stream.open_block(