You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
118 lines
3.4 KiB
118 lines
3.4 KiB
from toolz import concat, unique, count
|
|
|
|
from .compatibility import Mapping
|
|
|
|
import warnings
|
|
|
|
warnings.warn("ShareDict has been deprecated in favor of HighLevelGraph "
|
|
"and will be removed in future versions", stacklevel=2)
|
|
|
|
|
|
class ShareDict(Mapping):
|
|
""" A Mapping composed of other Mappings
|
|
|
|
This is a union of other disjoint mappings. It allows the combination of
|
|
many dicts into a single dict-like object without creating copies of the
|
|
underlying dicts. It provides cheap ``update``, ``len`` and ``__iter__``
|
|
operations as well as a fairly cheap ``__getitem__`` operation (linear in
|
|
the number of constituent mappings).
|
|
|
|
This class is optimized for Dask's use, and may not be generally useful.
|
|
Users may want to consider the standard ``collections.ChainMap`` data
|
|
structure.
|
|
|
|
This class makes the following assumptions:
|
|
|
|
1. Constituent mappings are disjoint. No key is in more than one mapping.
|
|
2. Constituent mappings will not be modified
|
|
|
|
Note that ShareDict does not enforce these assumptions. It is up to the
|
|
user to guarantee them.
|
|
|
|
Examples
|
|
--------
|
|
>>> a = {'x': 1, 'y': 2}
|
|
>>> b = {'z': 3}
|
|
>>> s = ShareDict()
|
|
>>> s.update(a)
|
|
>>> s.update(b)
|
|
|
|
>>> dict(s) # doctest: +SKIP
|
|
{'x': 1, 'y': 2, 'z': 3}
|
|
|
|
These dictionaries are stored within an internal dictionary of dictionaries
|
|
|
|
>>> list(s.dicts.values()) # doctest: +SKIP
|
|
[{'x': 1, 'y': 2}, {'z': 3}]
|
|
|
|
By default these are named by their object id. However, you can also
|
|
provide explicit names.
|
|
|
|
>>> s = ShareDict()
|
|
>>> s.update_with_key(a, key='a')
|
|
>>> s.update_with_key(b, key='b')
|
|
>>> s.dicts # doctest: +SKIP
|
|
{'a': {'x': 1, 'y': 2}, 'b': {'z': 3}}
|
|
"""
|
|
def __init__(self, dicts=None, dependencies=None):
|
|
self.dicts = dicts or dict()
|
|
self.dependencies = dependencies or dict()
|
|
|
|
assert set(self.dependencies) == set(self.dicts)
|
|
|
|
def update_with_key(self, arg, key=None, dependencies=None):
|
|
if type(arg) is ShareDict:
|
|
assert key is None or key in arg.dicts
|
|
self.dicts.update(arg.dicts)
|
|
self.dependencies.update(arg.dependencies)
|
|
return
|
|
|
|
if key is None:
|
|
key = id(arg)
|
|
|
|
if dependencies:
|
|
assert isinstance(dependencies, (tuple, list, set))
|
|
self.dependencies[key] = set(dependencies)
|
|
|
|
assert isinstance(arg, Mapping)
|
|
if arg:
|
|
self.dicts[key] = arg
|
|
|
|
def update(self, arg):
|
|
self.update_with_key(arg)
|
|
|
|
def __getitem__(self, key):
|
|
for d in self.dicts.values():
|
|
if key in d:
|
|
return d[key]
|
|
raise KeyError(key)
|
|
|
|
def __len__(self):
|
|
return count(iter(self))
|
|
|
|
def items(self):
|
|
seen = set()
|
|
for d in self.dicts.values():
|
|
for key in d:
|
|
if key not in seen:
|
|
seen.add(key)
|
|
yield (key, d[key])
|
|
|
|
def __iter__(self):
|
|
return unique(concat(self.dicts.values()))
|
|
|
|
|
|
def merge(*dicts, **kwargs):
|
|
dependencies = kwargs.pop('dependencies', None)
|
|
assert not kwargs
|
|
# assert dependencies is not None
|
|
result = ShareDict()
|
|
for d in dicts:
|
|
if isinstance(d, tuple):
|
|
key, d = d
|
|
result.update_with_key(d, key=key)
|
|
else:
|
|
result.update_with_key(d)
|
|
result.dependencies.update(dependencies or {})
|
|
return result
|