[Chandler-dev] Re: [commits] (vajda) [11956] - fixed bug 6895 again
(index check and repair were still broken)
Dan Steinicke
dan at osafoundation.org
Fri Oct 6 11:06:01 PDT 2006
Please don't commit when the tinderbox is not green (except to fix the
tree).
commits at osafoundation.org wrote:
>
> Revision
> 11956 <http://viewcvs.osafoundation.org/chandler?view=rev&rev=11956>
> Author
> vajda
> Date
> 2006-10-06 09:37:14 -0700 (Fri, 06 Oct 2006)
>
>
> Log Message
>
> - fixed bug 6895
> <http://bugzilla.osafoundation.org/show_bug.cgi?id=6895> again (index
> check and repair were still broken)
> (http://bugzilla.osafoundation.org/show_bug.cgi?id=6895)
> (--repair broken)
>
>
> Modified Paths
>
> * trunk/chandler/repository/item/Indexed.py
> <#trunkchandlerrepositoryitemIndexedpy>
> * trunk/chandler/repository/item/Indexes.py
> <#trunkchandlerrepositoryitemIndexespy>
> * trunk/chandler/repository/item/RefCollections.py
> <#trunkchandlerrepositoryitemRefCollectionspy>
> * trunk/chandler/repository/item/Sets.py
> <#trunkchandlerrepositoryitemSetspy>
> * trunk/chandler/repository/persistence/DBRefs.py
> <#trunkchandlerrepositorypersistenceDBRefspy>
> * trunk/chandler/repository/persistence/RepositoryView.py
> <#trunkchandlerrepositorypersistenceRepositoryViewpy>
>
>
> Diff
>
>
> Modified: trunk/chandler/repository/item/Indexed.py (11955 =>
> 11956)
>
> --- trunk/chandler/repository/item/Indexed.py 2006-10-06 00:04:06 UTC (rev 11955)
> +++ trunk/chandler/repository/item/Indexed.py 2006-10-06 16:37:14 UTC (rev 11956)
> @@ -289,10 +289,10 @@
> del self._indexes[indexName]
> self._setDirty(True) # noMonitors=True
>
> - def fillIndex(self, index):
> + def fillIndex(self, index, excludeIndexes=False):
>
> prevKey = None
> - for key in self.iterkeys():
> + for key in self.iterkeys(excludeIndexes):
> index.insertKey(key, prevKey)
> prevKey = key
>
> @@ -646,29 +646,25 @@
> result = True
>
> if self._indexes:
> - try:
> - indexes = self._indexes
> - self._indexes = None
> - count = len(self)
> + indexes = self._indexes
> + count = self.__len__(True)
>
> - for name, index in indexes.iteritems():
> - if not index._checkIndex(index, logger, name, self,
> - item, attribute, count, repair):
> - if repair:
> - logger.warning("Rebuilding index '%s' installed on value '%s' of type %s in attribute '%s' on %s", name, self, type(self), attribute, item._repr_())
> - kwds = index.getInitKeywords()
> - kwds.pop('ranges', None)
> - indexes[name] = index = \
> - self._createIndex(index.getIndexType(), **kwds)
> - self.fillIndex(index)
> - self._setDirty(True)
> + for name, index in indexes.iteritems():
> + if not index._checkIndex(index, logger, name, self,
> + item, attribute, count, repair):
> + if repair:
> + logger.warning("Rebuilding index '%s' installed on value '%s' of type %s in attribute '%s' on %s", name, self, type(self), attribute, item._repr_())
> + kwds = index.getInitKeywords()
> + kwds.pop('ranges', None)
> + indexes[name] = index = \
> + self._createIndex(index.getIndexType(), **kwds)
> + self.fillIndex(index, True)
> + self._setDirty(True)
>
> - result = index._checkIndex(index, logger, name,
> - self, item, attribute,
> - count, repair)
> - else:
> - result = False
> - finally:
> - self._indexes = indexes
> + result = index._checkIndex(index, logger, name,
> + self, item, attribute,
> + count, repair)
> + else:
> + result = False
>
> return result
>
>
> Modified: trunk/chandler/repository/item/Indexes.py (11955 =>
> 11956)
>
> --- trunk/chandler/repository/item/Indexes.py 2006-10-06 00:04:06 UTC (rev 11955)
> +++ trunk/chandler/repository/item/Indexes.py 2006-10-06 16:37:14 UTC (rev 11956)
> @@ -15,6 +15,7 @@
>
> from struct import pack, unpack
> from itertools import izip
> +from traceback import format_exc
>
> from chandlerdb.item.c import DelegatingIndex
> from chandlerdb.util.c import Nil, SkipList, CLinkedMap
> @@ -131,10 +132,15 @@
> result = False
>
> else:
> - size, result = _index._checkIterateIndex(logger, name, value,
> - item, attribute, repair)
> - if size != 0:
> - logger.error("Iteration of index '%s' (%d) installed on value '%s' of type %s in attribute '%s' on %s doesn't match length (%d)", name, count - size, value, type(value), attribute, item._repr_(), count)
> + try:
> + size, result = _index._checkIterateIndex(logger, name, value,
> + item, attribute,
> + repair)
> + if size != 0:
> + logger.error("Iteration of index '%s' (%d) installed on value '%s' of type %s in attribute '%s' on %s doesn't match length (%d)", name, count - size, value, type(value), attribute, item._repr_(), count)
> + result = False
> + except Exception, e:
> + logger.error("Iteration of index '%s' installed on value '%s' of type %s in attribute '%s' on %s caused an error: %s", name, value, type(value), attribute, item._repr_(), format_exc(5))
> result = False
>
> return result
>
>
> Modified: trunk/chandler/repository/item/RefCollections.py
> (11955 => 11956)
>
> --- trunk/chandler/repository/item/RefCollections.py 2006-10-06 00:04:06 UTC (rev 11955)
> +++ trunk/chandler/repository/item/RefCollections.py 2006-10-06 16:37:14 UTC (rev 11956)
> @@ -737,6 +737,14 @@
> return "\n%s<RefList> %s<->%s" %(' ' * indent,
> self._name, self._otherName)
>
> + def __iter__(self, excludeIndexes=False):
> +
> + for key in self.iterkeys():
> + yield self[key]
> +
> + def __len__(self, excludeIndexes=False):
> + return self._count
> +
> def iterItems(self):
> return self.itervalues()
>
>
>
> Modified: trunk/chandler/repository/item/Sets.py (11955 => 11956)
>
> --- trunk/chandler/repository/item/Sets.py 2006-10-06 00:04:06 UTC (rev 11955)
> +++ trunk/chandler/repository/item/Sets.py 2006-10-06 16:37:14 UTC (rev 11956)
> @@ -78,53 +78,56 @@
>
> return not self
>
> - def __iter__(self):
> + def __iter__(self, excludeIndexes=False):
>
> - index = self._anIndex()
> - if index is not None:
> - view = self._view
> - return (view[key] for key in index)
> + if not excludeIndexes:
> + index = self._anIndex()
> + if index is not None:
> + view = self._view
> + return (view[key] for key in index)
>
> - return self._itervalues()
> + return self._itervalues(excludeIndexes)
>
> - def itervalues(self):
> + def itervalues(self, excludeIndexes=False):
>
> - return self.__iter__()
> + return self.__iter__(excludeIndexes)
>
> - def _itervalues(self):
> + def _itervalues(self, excludeIndexes=False):
>
> raise NotImplementedError, "%s._itervalues" %(type(self))
>
> - def iterkeys(self):
> + def iterkeys(self, excludeIndexes=False):
>
> - index = self._anIndex()
> - if index is not None:
> - return index.iterkeys()
> + if not excludeIndexes:
> + index = self._anIndex()
> + if index is not None:
> + return index.iterkeys()
>
> - return self._iterkeys()
> + return self._iterkeys(excludeIndexes)
>
> # the slow way, via items, to be overridden by some implementations
> - def _iterkeys(self):
> + def _iterkeys(self, excludeIndexes=False):
>
> - return (item.itsUUID for item in self)
> + return (item.itsUUID for item in self.__iter__(excludeIndexes))
>
> def iterItems(self):
>
> return self.itervalues()
>
> - def __len__(self):
> + def __len__(self, excludeIndexes=False):
>
> - index = self._anIndex()
> - if index is not None:
> - return len(index)
> + if not excludeIndexes:
> + index = self._anIndex()
> + if index is not None:
> + return len(index)
>
> - return self._len()
> + return self._len(excludeIndexes)
>
> # the slow way, via keys, to be overridden by some implementations
> - def _len(self):
> + def _len(self, excludeIndexes=False):
>
> count = 0
> - for key in self.iterkeys():
> + for key in self.iterkeys(excludeIndexes):
> count += 1
>
> return count
> @@ -227,32 +230,30 @@
>
> return getattr(self._view[source[0]], source[1])._anIndex()
>
> - def _iterSource(self, source):
> + def _iterSource(self, source, excludeIndexes=False):
>
> if isinstance(source, AbstractSet):
> - for item in source:
> + for item in source.__iter__(excludeIndexes):
> yield item
> else:
> - for item in getattr(self._view[source[0]], source[1]):
> + for item in getattr(self._view[source[0]],
> + source[1]).__iter__(excludeIndexes):
> yield item
>
> - def _iterSourceKeys(self, source):
> + def _iterSourceKeys(self, source, excludeIndexes=False):
>
> if isinstance(source, AbstractSet):
> - return source.iterkeys()
> + return source.iterkeys(excludeIndexes)
>
> - return getattr(self._view[source[0]], source[1]).iterkeys()
> + return getattr(self._view[source[0]],
> + source[1]).iterkeys(excludeIndexes)
>
> - def _sourceLen(self, source):
> + def _sourceLen(self, source, excludeIndexes=False):
>
> if isinstance(source, AbstractSet):
> - return len(source)
> + return source.__len__(excludeIndexes)
>
> - try:
> - return len(getattr(self._view[source[0]], source[1]))
> - except AttributeError:
> - print source, type(source), self, type(self)
> - raise
> + return getattr(self._view[source[0]], source[1]).__len__(excludeIndexes)
>
> def _reprSource(self, source, replace):
>
> @@ -559,15 +560,15 @@
>
> return False
>
> - def _itervalues(self):
> + def _itervalues(self, excludeIndexes=False):
>
> return iter(())
>
> - def _iterkeys(self):
> + def _iterkeys(self, excludeIndexes=False):
>
> return iter(())
>
> - def _len(self):
> + def _len(self, excludeIndexes=False):
>
> return 0
>
> @@ -629,17 +630,17 @@
> return self._sourceContains(item, self._source,
> excludeMutating, excludeIndexes)
>
> - def _itervalues(self):
> + def _itervalues(self, excludeIndexes=False):
>
> - return self._iterSource(self._source)
> + return self._iterSource(self._source, excludeIndexes)
>
> - def _iterkeys(self):
> + def _iterkeys(self, excludeIndexes=False):
>
> - return self._iterSourceKeys(self._source)
> + return self._iterSourceKeys(self._source, excludeIndexes)
>
> - def _len(self):
> + def _len(self, excludeIndexes=True):
>
> - return self._sourceLen(self._source)
> + return self._sourceLen(self._source, excludeIndexes)
>
> def _repr_(self, replace=None):
>
> @@ -803,31 +804,33 @@
> self._sourceContains(item, self._right,
> excludeMutating, excludeIndexes))
>
> - def _itervalues(self):
> + def _itervalues(self, excludeIndexes=False):
>
> left = self._left
> - for item in self._iterSource(left):
> + for item in self._iterSource(left, excludeIndexes):
> yield item
> - for item in self._iterSource(self._right):
> - if not self._sourceContains(item, left):
> + for item in self._iterSource(self._right, excludeIndexes):
> + if not self._sourceContains(item, left, False, excludeIndexes):
> yield item
>
> - def _iterkeys(self):
> + def _iterkeys(self, excludeIndexes=False):
>
> - leftIndex = self._aSourceIndex(self._left)
> - if leftIndex is not None:
> - for key in leftIndex:
> - yield key
> - for key in self._iterSourceKeys(self._right):
> - if key not in leftIndex:
> + if not excludeIndexes:
> + leftIndex = self._aSourceIndex(self._left)
> + if leftIndex is not None:
> + for key in leftIndex:
> yield key
> - else:
> - for key in self._iterSourceKeys(self._left):
> + for key in self._iterSourceKeys(self._right):
> + if key not in leftIndex:
> + yield key
> + return
> +
> + for key in self._iterSourceKeys(self._left, excludeIndexes):
> + yield key
> + left = self._getSource(self._left)
> + for key in self._iterSourceKeys(self._right, excludeIndexes):
> + if not left.__contains__(key, False, excludeIndexes):
> yield key
> - left = self._getSource(self._left)
> - for key in self._iterSourceKeys(self._right):
> - if key not in left:
> - yield key
>
> def _op(self, leftOp, rightOp, other):
>
> @@ -861,28 +864,30 @@
> self._sourceContains(item, self._right,
> excludeMutating, excludeIndexes))
>
> - def _itervalues(self):
> + def _itervalues(self, excludeIndexes=False):
>
> left = self._left
> right = self._right
>
> - for item in self._iterSource(left):
> - if self._sourceContains(item, right):
> + for item in self._iterSource(left, excludeIndexes):
> + if self._sourceContains(item, right, False, excludeIndexes):
> yield item
>
> - def _iterkeys(self):
> + def _iterkeys(self, excludeIndexes=False):
>
> - rightIndex = self._aSourceIndex(self._right)
> - if rightIndex is not None:
> - for key in self._iterSourceKeys(self._left):
> - if key in rightIndex:
> - yield key
> - else:
> - right = self._getSource(self._right)
> - for key in self._iterSourceKeys(self._left):
> - if key in right:
> - yield key
> + if not excludeIndexes:
> + rightIndex = self._aSourceIndex(self._right)
> + if rightIndex is not None:
> + for key in self._iterSourceKeys(self._left):
> + if key in rightIndex:
> + yield key
> + return
>
> + right = self._getSource(self._right)
> + for key in self._iterSourceKeys(self._left, excludeIndexes):
> + if right.__contains__(key, False, excludeIndexes):
> + yield key
> +
> def _op(self, leftOp, rightOp, other):
>
> left = self._left
> @@ -915,28 +920,30 @@
> not self._sourceContains(item, self._right,
> excludeMutating, excludeIndexes))
>
> - def _itervalues(self):
> + def _itervalues(self, excludeIndexes=False):
>
> left = self._left
> right = self._right
>
> - for item in self._iterSource(left):
> - if not self._sourceContains(item, right):
> + for item in self._iterSource(left, excludeIndexes):
> + if not self._sourceContains(item, right, False, excludeIndexes):
> yield item
>
> - def _iterkeys(self):
> + def _iterkeys(self, excludeIndexes=False):
>
> - rightIndex = self._aSourceIndex(self._right)
> - if rightIndex is not None:
> - for key in self._iterSourceKeys(self._left):
> - if key not in rightIndex:
> - yield key
> - else:
> - right = self._getSource(self._right)
> - for key in self._iterSourceKeys(self._left):
> - if key not in right:
> - yield key
> + if not excludeIndexes:
> + rightIndex = self._aSourceIndex(self._right)
> + if rightIndex is not None:
> + for key in self._iterSourceKeys(self._left):
> + if key not in rightIndex:
> + yield key
> + return
>
> + right = self._getSource(self._right)
> + for key in self._iterSourceKeys(self._left, excludeIndexes):
> + if not right.__contains__(key, False, excludeIndexes):
> + yield key
> +
> def _op(self, leftOp, rightOp, other):
>
> left = self._left
> @@ -1058,31 +1065,31 @@
>
> return False
>
> - def _iterkeys(self):
> + def _iterkeys(self, excludeIndexes=False):
>
> sources = self._sources
> for source in sources:
> - for key in self._iterSourceKeys(source):
> + for key in self._iterSourceKeys(source, excludeIndexes):
> unique = True
> for src in sources:
> if src is source:
> break
> - if self._sourceContains(key, src):
> + if self._sourceContains(key, src, False, excludeIndexes):
> unique = False
> break
> if unique:
> yield key
>
> - def _itervalues(self):
> + def _itervalues(self, excludeIndexes=False):
>
> sources = self._sources
> for source in sources:
> - for item in self._iterSource(source):
> + for item in self._iterSource(source, excludeIndexes):
> unique = True
> for src in sources:
> if src is source:
> break
> - if self._sourceContains(item, src):
> + if self._sourceContains(item, src, False, excludeIndexes):
> unique = False
> break
> if unique:
> @@ -1125,33 +1132,35 @@
>
> return True
>
> - def _iterkeys(self):
> + def _iterkeys(self, excludeIndexes=False):
>
> sources = self._sources
> if len(sources) > 1:
> source = sources[0]
> - for key in self._iterSourceKeys(source):
> + for key in self._iterSourceKeys(source, excludeIndexes):
> everywhere = True
> for src in sources:
> if src is source:
> continue
> - if not self._sourceContains(key, src):
> + if not self._sourceContains(key, src,
> + False, excludeIndexes):
> everywhere = False
> break
> if everywhere:
> yield key
>
> - def _itervalues(self):
> + def _itervalues(self, excludeIndexes=False):
>
> sources = self._sources
> if len(sources) > 1:
> source = sources[0]
> - for item in self._iterSource(source):
> + for item in self._iterSource(source, excludeIndexes):
> everywhere = True
> for src in sources:
> if src is source:
> continue
> - if not self._sourceContains(item, src):
> + if not self._sourceContains(item, src,
> + False, excludeIndexes):
> everywhere = False
> break
> if everywhere:
> @@ -1224,11 +1233,11 @@
>
> return self.__contains__(item, excludeMutating, excludeIndexes)
>
> - def _itervalues(self):
> + def _itervalues(self, excludeIndexes=False):
>
> return self._view[self._extent].iterItems(self._recursive)
>
> - def _iterkeys(self):
> + def _iterkeys(self, excludeIndexes=False):
>
> return self._view[self._extent].iterKeys(self._recursive)
>
> @@ -1254,9 +1263,9 @@
>
> return op
>
> - def _len(self):
> + def _len(self, excludeIndexes=False):
>
> - return AbstractSet._len(self)
> + return AbstractSet._len(self, excludeIndexes)
>
> def iterSources(self):
>
> @@ -1296,22 +1305,22 @@
>
> return False
>
> - def _iterkeys(self):
> + def _iterkeys(self, excludeIndexes=False):
>
> - for uuid in self._iterSourceKeys(self._source):
> + for uuid in self._iterSourceKeys(self._source, excludeIndexes):
> if self.filter(uuid):
> yield uuid
>
> - def _itervalues(self):
> + def _itervalues(self, excludeIndexes=False):
>
> - for item in self._iterSource(self._source):
> + for item in self._iterSource(self._source, excludeIndexes):
> if self.filter(item.itsUUID):
> yield item
>
> - def _len(self):
> + def _len(self, excludeIndexes=False):
>
> count = 0
> - for key in self._iterkeys():
> + for key in self._iterkeys(excludeIndexes):
> count += 1
>
> return count
>
>
> Modified: trunk/chandler/repository/persistence/DBRefs.py
> (11955 => 11956)
>
> --- trunk/chandler/repository/persistence/DBRefs.py 2006-10-06 00:04:06 UTC (rev 11955)
> +++ trunk/chandler/repository/persistence/DBRefs.py 2006-10-06 16:37:14 UTC (rev 11956)
> @@ -321,7 +321,7 @@
> RefList.__init__(self, item, name, otherName, dictKey, readOnly,
> (new and CLinkedMap.NEW or 0) | CLinkedMap.LOAD)
>
> - def iterkeys(self, firstKey=None, lastKey=None):
> + def iterkeys(self, excludeIndexes=False, firstKey=None, lastKey=None):
>
> return self._iterrefs(firstKey, lastKey)
>
>
>
> Modified:
> trunk/chandler/repository/persistence/RepositoryView.py (11955
> => 11956)
>
> --- trunk/chandler/repository/persistence/RepositoryView.py 2006-10-06 00:04:06 UTC (rev 11955)
> +++ trunk/chandler/repository/persistence/RepositoryView.py 2006-10-06 16:37:14 UTC (rev 11956)
> @@ -1610,6 +1610,9 @@
> def _setDirty(self, noMonitors=False):
> pass
>
> + def iterkeys(self, excludeIndexes=False, firstKey=None, lastKey=None):
> + return super(NullViewRefList, self).iterkeys(firstKey, lastKey)
> +
> def _unloadRef(self, item):
>
> key = item.itsUUID
>
>
> ------------------------------------------------------------------------
>
> _______________________________________________
> Commits mailing list
> Commits at osafoundation.org
> http://lists.osafoundation.org/mailman/listinfo/commits
>
More information about the chandler-dev
mailing list