[Commits] (vajda) - started implementation of cloud serializer
commits at osafoundation.org
commits at osafoundation.org
Thu May 13 23:35:00 PDT 2004
Commit by: vajda
Modified files:
chandler/repository/item/ItemHandler.py 1.36 1.37
chandler/repository/persistence/DBContainer.py 1.9 1.10
chandler/repository/persistence/Repository.py 1.70 1.71
chandler/repository/persistence/XMLRepository.py 1.70 1.71
chandler/repository/persistence/XMLRepositoryView.py 1.33 1.34
chandler/repository/persistence/RemoteRepository.py 1.10 None
chandler/repository/persistence/RepositoryServer.py 1.6 None
chandler/repository/persistence/Transport.py 1.9 None
chandler/repository/remote/.cvsignore None 1.1
chandler/repository/remote/CloudFilter.py None 1.1
chandler/repository/remote/ItemFilter.py None 1.1
chandler/repository/remote/RemoteFilter.py None 1.1
chandler/repository/remote/RemoteRepository.py None 1.1
chandler/repository/remote/RepositoryServer.py None 1.1
chandler/repository/remote/Transport.py None 1.1
chandler/repository/remote/__init__.py None 1.1
chandler/repository/schema/Cloud.py 1.1 1.2
chandler/repository/schema/Types.py 1.57 1.58
chandler/repository/util/SAX.py 1.3 1.4
chandler/repository/util/ext/.cvsignore 1.3 None
chandler/repository/util/ext/pyuuid.c 1.4 None
chandler/repository/util/ext/setup.py 1.3 None
chandler/repository/util/ext/uuid.c 1.4 None
chandler/repository/util/ext/uuid.h 1.2 None
Log message:
- started implementation of cloud serializer
- continued on cloud serializer
- added XMLPrettyGenerator, an XML pretty printer
- continued on cloud serializer, added item serializer
- added Text support to item serializer
- continued on cloud serializer, added endpoint and cloud recursion
- continued on cloud serializer
- started on remoting experiment again
- continued on remoting experiment again
- remote repository serves xml again over SOAP
- refactored low-level ref collection code for remote cloud import
- refactored low-level item save code for remote cloud import
- refactored low-level lob code for remote cloud import
- got first item transfered
- remove UUIDext code from repository dir, it's in internal now
- remote access is revived, items are transfered and accessible as such
ViewCVS links:
http://cvs.osafoundation.org/index.cgi/chandler/repository/item/ItemHandler.py.diff?r1=text&tr1=1.36&r2=text&tr2=1.37
http://cvs.osafoundation.org/index.cgi/chandler/repository/persistence/DBContainer.py.diff?r1=text&tr1=1.9&r2=text&tr2=1.10
http://cvs.osafoundation.org/index.cgi/chandler/repository/persistence/Repository.py.diff?r1=text&tr1=1.70&r2=text&tr2=1.71
http://cvs.osafoundation.org/index.cgi/chandler/repository/persistence/XMLRepository.py.diff?r1=text&tr1=1.70&r2=text&tr2=1.71
http://cvs.osafoundation.org/index.cgi/chandler/repository/persistence/XMLRepositoryView.py.diff?r1=text&tr1=1.33&r2=text&tr2=1.34
http://cvs.osafoundation.org/index.cgi/chandler/repository/persistence/RemoteRepository.py.diff?r1=text&tr1=1.10&r2=text&tr2=None
http://cvs.osafoundation.org/index.cgi/chandler/repository/persistence/RepositoryServer.py.diff?r1=text&tr1=1.6&r2=text&tr2=None
http://cvs.osafoundation.org/index.cgi/chandler/repository/persistence/Transport.py.diff?r1=text&tr1=1.9&r2=text&tr2=None
http://cvs.osafoundation.org/index.cgi/chandler/repository/remote/.cvsignore?rev=1.1&content-type=text/vnd.viewcvs-markup
http://cvs.osafoundation.org/index.cgi/chandler/repository/remote/CloudFilter.py?rev=1.1&content-type=text/vnd.viewcvs-markup
http://cvs.osafoundation.org/index.cgi/chandler/repository/remote/ItemFilter.py?rev=1.1&content-type=text/vnd.viewcvs-markup
http://cvs.osafoundation.org/index.cgi/chandler/repository/remote/RemoteFilter.py?rev=1.1&content-type=text/vnd.viewcvs-markup
http://cvs.osafoundation.org/index.cgi/chandler/repository/remote/RemoteRepository.py?rev=1.1&content-type=text/vnd.viewcvs-markup
http://cvs.osafoundation.org/index.cgi/chandler/repository/remote/RepositoryServer.py?rev=1.1&content-type=text/vnd.viewcvs-markup
http://cvs.osafoundation.org/index.cgi/chandler/repository/remote/Transport.py?rev=1.1&content-type=text/vnd.viewcvs-markup
http://cvs.osafoundation.org/index.cgi/chandler/repository/remote/__init__.py?rev=1.1&content-type=text/vnd.viewcvs-markup
http://cvs.osafoundation.org/index.cgi/chandler/repository/schema/Cloud.py.diff?r1=text&tr1=1.1&r2=text&tr2=1.2
http://cvs.osafoundation.org/index.cgi/chandler/repository/schema/Types.py.diff?r1=text&tr1=1.57&r2=text&tr2=1.58
http://cvs.osafoundation.org/index.cgi/chandler/repository/util/SAX.py.diff?r1=text&tr1=1.3&r2=text&tr2=1.4
http://cvs.osafoundation.org/index.cgi/chandler/repository/util/ext/.cvsignore.diff?r1=text&tr1=1.3&r2=text&tr2=None
http://cvs.osafoundation.org/index.cgi/chandler/repository/util/ext/pyuuid.c.diff?r1=text&tr1=1.4&r2=text&tr2=None
http://cvs.osafoundation.org/index.cgi/chandler/repository/util/ext/setup.py.diff?r1=text&tr1=1.3&r2=text&tr2=None
http://cvs.osafoundation.org/index.cgi/chandler/repository/util/ext/uuid.c.diff?r1=text&tr1=1.4&r2=text&tr2=None
http://cvs.osafoundation.org/index.cgi/chandler/repository/util/ext/uuid.h.diff?r1=text&tr1=1.2&r2=text&tr2=None
Index: chandler/repository/item/ItemHandler.py
diff -u chandler/repository/item/ItemHandler.py:1.36 chandler/repository/item/ItemHandler.py:1.37
--- chandler/repository/item/ItemHandler.py:1.36 Fri Apr 9 12:22:36 2004
+++ chandler/repository/item/ItemHandler.py Thu May 13 23:34:22 2004
@@ -1,6 +1,6 @@
-__revision__ = "$Revision: 1.36 $"
-__date__ = "$Date: 2004/04/09 19:22:36 $"
+__revision__ = "$Revision: 1.37 $"
+__date__ = "$Date: 2004/05/14 06:34:22 $"
__copyright__ = "Copyright (c) 2002 Open Source Applications Foundation"
__license__ = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
@@ -303,7 +303,7 @@
raise ValueError, self.tagAttrs[-1]['name']
refDict = self.collections[-1]
- refDict._prepareKey(self.uuid, UUID(self.data))
+ refDict._prepareBuffers(self.uuid, UUID(self.data))
def attributeStart(self, itemHandler, attrs):
@@ -424,6 +424,12 @@
return self.repository[UUID(attrs['typeid'])].handlerName()
except KeyError:
raise TypeError, "Type %s not found" %(attrs['typeid'])
+
+ if attrs.has_key('typepath'):
+ typeItem = self.repository.find(Path(attrs['typepath']))
+ if typeItem is None:
+ raise TypeError, "Type %s not found" %(attrs['typepath'])
+ return typeItem.handlerName()
if attrs.has_key('type'):
return attrs['type']
Index: chandler/repository/persistence/XMLRepositoryView.py
diff -u chandler/repository/persistence/XMLRepositoryView.py:1.33 chandler/repository/persistence/XMLRepositoryView.py:1.34
--- chandler/repository/persistence/XMLRepositoryView.py:1.33 Fri Apr 9 12:22:37 2004
+++ chandler/repository/persistence/XMLRepositoryView.py Thu May 13 23:34:23 2004
@@ -1,6 +1,6 @@
-__revision__ = "$Revision: 1.33 $"
-__date__ = "$Date: 2004/04/09 19:22:37 $"
+__revision__ = "$Revision: 1.34 $"
+__date__ = "$Date: 2004/05/14 06:34:23 $"
__copyright__ = "Copyright (c) 2002 Open Source Applications Foundation"
__license__ = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
@@ -10,8 +10,6 @@
from struct import pack
from bsddb.db import DBLockDeadlockError, DBNotFoundError
-from bsddb.db import DB_DIRTY_READ, DB_LOCK_WRITE
-from dbxml import XmlDocument, XmlValue
from repository.item.Item import Item
from repository.item.Values import Values, ItemValue
@@ -34,6 +32,7 @@
self._log = []
self._notifications = RepositoryNotifications(repository)
+ self._indexWriter = None
def logItem(self, item):
@@ -97,16 +96,6 @@
return results
-
-class XMLRepositoryLocalView(XMLRepositoryView):
-
- def __init__(self, repository):
-
- super(XMLRepositoryLocalView, self).__init__(repository)
-
- self._indexWriter = None
- self._lock_id = self.repository._env.lock_id()
-
def createRefDict(self, item, name, otherName, persist):
if persist:
@@ -125,7 +114,7 @@
def _startTransaction(self):
- return self.repository.store._startTransaction()
+ return self.repository.store.startTransaction()
def _commitTransaction(self):
@@ -134,7 +123,7 @@
self._indexWriter.close()
self._indexWriter = None
- self.repository.store._commitTransaction()
+ self.repository.store.commitTransaction()
def _abortTransaction(self):
@@ -142,7 +131,7 @@
self._indexWriter.close()
self._indexWriter = None
- self.repository.store._abortTransaction()
+ self.repository.store.abortTransaction()
def _getIndexWriter(self):
@@ -158,11 +147,8 @@
repository = self.repository
store = repository.store
- data = store._data
- names = store._names
versions = store._versions
history = store._history
- env = repository._env
self._notifications.clear()
histNotifications = None
@@ -180,10 +166,9 @@
newVersion = versions.getVersion()
if count > 0:
- lock = env.lock_get(self._lock_id, self.itsUUID._uuid,
- DB_LOCK_WRITE)
+ lock = store.acquireLock()
newVersion += 1
- versions.put(self.itsUUID._uuid, pack('>l', ~newVersion))
+ versions.setVersion(newVersion)
ood = {}
for item in self._log:
@@ -199,9 +184,7 @@
history)
for item in self._log:
- size += self._saveItem(item, newVersion,
- data, names, versions, history,
- ood)
+ size += self._saveItem(item, newVersion, store, ood)
if newVersion > self.version:
histNotifications = RepositoryNotifications(repository)
@@ -228,8 +211,7 @@
self._commitTransaction()
if lock:
- env.lock_put(lock)
- lock = None
+ lock = store.releaseLock(lock)
break
@@ -238,8 +220,7 @@
if txnStarted:
self._abortTransaction()
if lock:
- env.lock_put(lock)
- lock = None
+ lock = store.releaseLock(lock)
continue
@@ -248,8 +229,7 @@
if txnStarted:
self._abortTransaction()
if lock:
- env.lock_put(lock)
- lock = None
+ lock = store.releaseLock(lock)
raise
@@ -282,7 +262,7 @@
datetime.now() - before)
self.prune(10000)
- def _saveItem(self, item, newVersion, data, names, versions, history, ood):
+ def _saveItem(self, item, newVersion, store, ood):
uuid = item._uuid
isNew = item.isNew()
@@ -300,7 +280,7 @@
if uuid in ood:
docId, oldDirty, newDirty = ood[uuid]
- mergeWith = (data.getDocument(docId).getContent(),
+ mergeWith = (store._data.getDocument(docId).getContent(),
oldDirty, newDirty)
if isDebug:
self.logger.debug('merging %s (%0.4x:%0.4x) with newest version',
@@ -313,47 +293,27 @@
generator.startDocument()
item._saveItem(generator, newVersion, mergeWith)
generator.endDocument()
- content = out.getvalue()
+ xml = out.getvalue()
out.close()
- size = len(content)
-
- doc = XmlDocument()
- doc.setContent(content)
- if isDeleted:
- doc.setMetaData('', '', 'deleted', XmlValue('True'))
-
- try:
- docId = data.putDocument(doc)
- except:
- self.logger.exception("putDocument failed, xml is: %s", content)
- raise
+ if '_origName' in item.__dict__:
+ origPN = item.__dict__['_origName']
+ del item.__dict__['_origName']
+ else:
+ origPN = None
+
+ store.saveItem(xml, uuid, newVersion,
+ (item.itsParent.itsUUID, item._name), origPN,
+ item._status)
if isDeleted:
- parent, name = item.__dict__['_origName']
- versions.setDocVersion(uuid, newVersion, 0)
- history.writeVersion(uuid, newVersion, 0, item._status, parent)
- self._notifications.changed(uuid, 'deleted', parent=parent)
- names.writeName(parent, name, newVersion, None)
-
+ self._notifications.changed(uuid, 'deleted', parent=origPN[0])
+ elif isNew:
+ self._notifications.changed(uuid, 'added')
else:
- versions.setDocVersion(uuid, newVersion, docId)
- history.writeVersion(uuid, newVersion, docId, item._status)
-
- if isNew:
- names.writeName(item.itsParent.itsUUID, item._name,
- newVersion, uuid)
- self._notifications.changed(uuid, 'added')
- else:
- if '_origName' in item.__dict__:
- parent, name = item.__dict__['_origName']
- names.writeName(parent, name, newVersion, None)
- names.writeName(item.itsParent.itsUUID, item._name,
- newVersion, uuid)
- del item.__dict__['_origName']
- self._notifications.changed(uuid, 'changed')
+ self._notifications.changed(uuid, 'changed')
- return size
+ return len(xml)
def _mergeItems(self, items, oldVersion, newVersion, history):
@@ -375,16 +335,6 @@
history.apply(check, oldVersion, newVersion)
-class XMLRepositoryClientView(XMLRepositoryView):
-
- def createRefDict(self, item, name, otherName, persist):
-
- if persist:
- return XMLClientRefDict(self, item, name, otherName)
- else:
- return TransientRefDict(item, name, otherName)
-
-
class XMLRefDict(RefDict):
class _log(list):
@@ -408,6 +358,10 @@
return self.view
+ def _getRefs(self):
+
+ return self.view.repository.store._refs
+
def _loadRef(self, key):
view = self.view
@@ -418,15 +372,7 @@
if key in self._deletedRefs:
return None
- return self._loadRef_(key)
-
- def _loadRef_(self, key):
-
- version = self._item._version
- cursorKey = self._packKey(key)
-
- return self.view.repository.store._refs.loadRef(version, key,
- cursorKey)
+ return self._getRefs().loadRef(self._key, self._item._version, key)
def _changeRef(self, key, alias=None):
@@ -448,47 +394,12 @@
def _writeRef(self, key, version, uuid, previous, next, alias):
- self._value.truncate(0)
- self._value.seek(0)
- if uuid is not None:
- self._writeValue(uuid)
- self._writeValue(previous)
- self._writeValue(next)
- self._writeValue(alias)
- else:
- self._writeValue(None)
- value = self._value.getvalue()
-
- self.view.repository.store._refs.put(self._packKey(key, version),
- value)
-
- def _writeValue(self, value):
-
- if isinstance(value, UUID):
- self._value.write('\0')
- self._value.write(value._uuid)
-
- elif isinstance(value, str):
- self._value.write('\1')
- self._value.write(pack('>H', len(value)))
- self._value.write(value)
-
- elif isinstance(value, unicode):
- value = value.encode('utf-8')
- self._value.write('\1')
- self._value.write(pack('>H', len(value)))
- self._value.write(value)
-
- elif value is None:
- self._value.write('\2')
-
- else:
- raise NotImplementedError, "value: %s, type: %s" %(value,
- type(value))
+ self._getRefs().saveRef(self._key, self._value, key, version,
+ uuid, previous, next, alias)
def _eraseRef(self, key):
- self.view.repository.store._refs.delete(self._packKey(key))
+ self._getRefs().eraseRef(self._key, key)
def resolveAlias(self, alias):
@@ -511,28 +422,14 @@
self._item = item
if item is not None:
- self._prepareKey(item._uuid, self._uuid)
-
- def _packKey(self, key, version=None):
-
- self._key.truncate(32)
- self._key.seek(0, 2)
- self._key.write(key._uuid)
- if version is not None:
- self._key.write(pack('>l', ~version))
+ self._prepareBuffers(item._uuid, self._uuid)
- return self._key.getvalue()
-
- def _prepareKey(self, uItem, uuid):
+ def _prepareBuffers(self, uItem, uuid):
self._uuid = uuid
+ self._key = self._getRefs().prepareKey(uItem, uuid)
+ self._value = cStringIO.StringIO()
- self._key = cStringIO.StringIO()
- self._key.write(uItem._uuid)
- self._key.write(uuid._uuid)
-
- self._value = cStringIO.StringIO()
-
def _xmlValues(self, generator, version, mode):
if mode == 'save':
@@ -586,22 +483,6 @@
raise ValueError, mode
-class XMLClientRefDict(XMLRefDict):
-
- def _prepareKey(self, uItem, uuid):
-
- self._uItem = uItem
- self._uuid = uuid
-
- def _loadRef_(self, key):
-
- return self.view.repository.store.loadRef(self._item._version,
- self._uItem, self._uuid, key)
-
- def _writeRef(self, key, version, uuid, previous, next, alias):
- raise NotImplementedError, "XMLClientRefDict._writeRef"
-
-
class XMLText(Text, ItemValue):
def __init__(self, view, *args, **kwds):
@@ -620,10 +501,12 @@
if self._dirty:
store = self._view.repository.store
if self._append:
- out = store._text.appendFile(self._makeKey())
+ out = store._text.appendFile(store.lobName(uuid,
+ self._version))
else:
self._version += 1
- out = store._text.createFile(self._makeKey())
+ out = store._text.createFile(store.lobName(uuid,
+ self._version))
out.write(self._data)
out.close()
self._data = ''
@@ -663,11 +546,7 @@
return self._version
- def _makeKey(self):
-
- return pack('>16sl', self.getUUID()._uuid, ~self._version)
-
- def _textEnd(self, data, attrs):
+ def load(self, data, attrs):
self.mimetype = attrs.get('mimetype', 'text/plain')
self._compression = attrs.get('compression', None)
@@ -696,8 +575,9 @@
else:
dataIn = None
- text = self._view.repository.store._text
- key = self._makeKey()
+ store = self._view.repository.store
+ text = store._text
+ key = store.lobName(self.getUUID(), self._version)
if dataIn is not None:
if self._append:
@@ -731,10 +611,12 @@
if self._dirty:
store = self._view.repository.store
if self._append:
- out = store._binary.appendFile(self._makeKey())
+ out = store._binary.appendFile(store.lobName(uuid,
+ self._version))
else:
self._version += 1
- out = store._binary.createFile(self._makeKey())
+ out = store._binary.createFile(store.lobName(uuid,
+ self._version))
out.write(self._data)
out.close()
@@ -763,11 +645,7 @@
return self._version
- def _makeKey(self):
-
- return pack('>16sl', self._uuid._uuid, ~self._version)
-
- def _binaryEnd(self, data, attrs):
+ def load(self, data, attrs):
self.mimetype = attrs.get('mimetype', 'text/plain')
self._compression = attrs.get('compression', None)
@@ -792,8 +670,9 @@
else:
dataIn = None
- binary = self._view.repository.store._binary
- key = self._makeKey()
+ store = self._view.repository.store
+ binary = store._binary
+ key = store.lobName(self.getUUID(), self._version)
if dataIn is not None:
if self._append:
Index: chandler/repository/schema/Cloud.py
diff -u chandler/repository/schema/Cloud.py:1.1 chandler/repository/schema/Cloud.py:1.2
--- chandler/repository/schema/Cloud.py:1.1 Fri Apr 23 11:21:37 2004
+++ chandler/repository/schema/Cloud.py Thu May 13 23:34:26 2004
@@ -1,12 +1,16 @@
-__revision__ = "$Revision: 1.1 $"
-__date__ = "$Date: 2004/04/23 18:21:37 $"
+__revision__ = "$Revision: 1.2 $"
+__date__ = "$Date: 2004/05/14 06:34:26 $"
__copyright__ = "Copyright (c) 2002 Open Source Applications Foundation"
__license__ = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
+import libxml2, re
from repository.item.Item import Item
from repository.item.ItemRef import RefDict
+from repository.remote.CloudFilter import CloudFilter, EndpointFilter
+from repository.remote.CloudFilter import RefHandler
+from repository.persistence.Repository import NoSuchItemError
class Cloud(Item):
@@ -47,6 +51,35 @@
return items
+ def getEndpoints(self, name, index=0):
+
+ endpoints = []
+ for endpoint in self.endpoints:
+ names = endpoint.attribute
+ if index < len(names) and names[index] == name:
+ endpoints.append(endpoint)
+
+ return endpoints
+
+ def writeItems(self, uuid, version, generator, xml=None, uuids=None):
+
+ if uuids is None:
+ uuids = {}
+
+ if not uuid in uuids:
+ store = self.getRepository().store
+
+ uuids[uuid] = uuid
+ if xml is None:
+ doc = store.loadItem(version, uuid)
+ if doc is None:
+ raise NoSuchItemError, (uuid, version)
+
+ xml = doc.getContent()
+
+ filter = CloudFilter(self, store, uuid, version, generator)
+ filter.parse(xml, uuids)
+
class Endpoint(Item):
@@ -60,14 +93,59 @@
elif policy == 'byCloud':
cloud = self.getAttributeValue('cloud', default=None,
_attrDict=self._references)
-
if cloud is None:
- cloud = item.itsKind.getClouds()
+ cloud = kind.getClouds()
if not cloud:
- raise TypeError, 'No cloud for %s' %(item.itsKind.itsPath)
+ raise TypeError, 'No cloud for %s' %(kind.itsPath)
cloud = cloud[0]
cloud.getItems(item, items)
else:
raise NotImplementedError, policy
+
+ def writeItems(self, index, uuid, version, generator, xml, uuids):
+
+ names = self.attribute
+
+ if index == len(names):
+ if not uuid in uuids:
+ policy = self.includePolicy
+
+ if policy == 'byValue':
+ filter = EndpointFilter(self, self.getRepository().store,
+ uuid, version, generator)
+ filter.parse(xml)
+ uuids[uuid] = uuid
+
+ elif policy == 'byCloud':
+ cloud = self.getAttributeValue('cloud', default=None,
+ _attrDict=self._references)
+ if cloud is None:
+ match = self.kindExp.match(xml, xml.index("<kind "))
+ kind = self.getRepository()[UUID(match.group(1))]
+ cloud = kind.getClouds()
+ if not cloud:
+ raise TypeError, 'No cloud for %s' %(kind.itsPath)
+ cloud = cloud[0]
+
+ cloud.writeItems(uuid, version, generator, xml, uuids)
+
+ else:
+ raise NotImplementedError, policy
+
+ else:
+ handler = RefHandler(self, names[index], uuid, version)
+ handler.parse(xml)
+
+ if handler.values is not None:
+ store = self.getRepository().repository.store
+ for uuid in handler.values:
+ doc = store.loadItem(version, uuid)
+ if doc is None:
+ raise NoSuchItemError, (uuid, version)
+ xml = doc.getContent()
+ self.writeItems(index + 1, uuid, version, generator,
+ xml, uuids)
+
+ kindExp = re.compile('<kind type="uuid">(.*)</kind>')
Index: chandler/repository/persistence/DBContainer.py
diff -u chandler/repository/persistence/DBContainer.py:1.9 chandler/repository/persistence/DBContainer.py:1.10
--- chandler/repository/persistence/DBContainer.py:1.9 Fri Apr 9 12:22:37 2004
+++ chandler/repository/persistence/DBContainer.py Thu May 13 23:34:22 2004
@@ -1,10 +1,10 @@
-__revision__ = "$Revision: 1.9 $"
-__date__ = "$Date: 2004/04/09 19:22:37 $"
+__revision__ = "$Revision: 1.10 $"
+__date__ = "$Date: 2004/05/14 06:34:22 $"
__copyright__ = "Copyright (c) 2002 Open Source Applications Foundation"
__license__ = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
-import UUIDext
+import UUIDext, cStringIO
from struct import pack, unpack
@@ -74,6 +74,24 @@
class RefContainer(DBContainer):
+ def prepareKey(self, uItem, uuid):
+
+ buffer = cStringIO.StringIO()
+ buffer.write(uItem._uuid)
+ buffer.write(uuid._uuid)
+
+ return buffer
+
+ def _packKey(self, buffer, key, version=None):
+
+ buffer.truncate(32)
+ buffer.seek(0, 2)
+ buffer.write(key._uuid)
+ if version is not None:
+ buffer.write(pack('>l', ~version))
+
+ return buffer.getvalue()
+
def _readValue(self, value, offset):
code = value[offset]
@@ -92,14 +110,59 @@
raise ValueError, code
- def loadRef(self, version, key, cursorKey):
+ def _writeValue(self, buffer, value):
+
+ if isinstance(value, UUID):
+ buffer.write('\0')
+ buffer.write(value._uuid)
+
+ elif isinstance(value, str):
+ buffer.write('\1')
+ buffer.write(pack('>H', len(value)))
+ buffer.write(value)
+
+ elif isinstance(value, unicode):
+ value = value.encode('utf-8')
+ buffer.write('\1')
+ buffer.write(pack('>H', len(value)))
+ buffer.write(value)
+
+ elif value is None:
+ buffer.write('\2')
+
+ else:
+ raise NotImplementedError, "value: %s, type: %s" %(value,
+ type(value))
+
+ def saveRef(self, keyBuffer, buffer, key, version,
+ uuid, previous, next, alias):
+
+ buffer.truncate(0)
+ buffer.seek(0)
+ if uuid is not None:
+ self._writeValue(buffer, uuid)
+ self._writeValue(buffer, previous)
+ self._writeValue(buffer, next)
+ self._writeValue(buffer, alias)
+ else:
+ self._writeValue(buffer, None)
+
+ self.put(self._packKey(keyBuffer, key, version), buffer.getvalue())
+
+ def eraseRef(self, buffer, key):
+
+ self.delete(self._packKey(buffer, key))
+
+ def loadRef(self, buffer, version, key):
+
+ cursorKey = self._packKey(buffer, key)
while True:
txnStarted = False
cursor = None
try:
- txnStarted = self.store._startTransaction()
+ txnStarted = self.store.startTransaction()
cursor = self.cursor()
try:
@@ -155,8 +218,8 @@
if cursor:
cursor.close()
if txnStarted:
- self.store._abortTransaction()
-
+ self.store.abortTransaction()
+
# has to run within the commit() transaction
def deleteItem(self, item):
@@ -191,6 +254,10 @@
return ~unpack('>l', self.get(Repository.itsUUID._uuid))[0]
+ def setVersion(self, version):
+
+ self.put(Repository.itsUUID._uuid, pack('>l', ~version))
+
def setDocVersion(self, uuid, version, docId):
self.put(pack('>16sl', uuid._uuid, ~version), pack('>l', docId))
@@ -202,7 +269,7 @@
cursor = None
try:
- txnStarted = self.store._startTransaction()
+ txnStarted = self.store.startTransaction()
cursor = self.cursor()
try:
@@ -239,7 +306,7 @@
if cursor:
cursor.close()
if txnStarted:
- self.store._abortTransaction()
+ self.store.abortTransaction()
def getDocId(self, uuid, version):
@@ -248,7 +315,7 @@
cursor = None
try:
- txnStarted = self.store._startTransaction()
+ txnStarted = self.store.startTransaction()
cursor = self.cursor()
try:
@@ -285,7 +352,7 @@
if cursor:
cursor.close()
if txnStarted:
- self.store._abortTransaction()
+ self.store.abortTransaction()
def deleteVersion(self, uuid):
@@ -360,7 +427,7 @@
cursor = None
try:
- txnStarted = self.store._startTransaction()
+ txnStarted = self.store.startTransaction()
cursor = self.cursor()
try:
@@ -400,4 +467,4 @@
if cursor:
cursor.close()
if txnStarted:
- self.store._abortTransaction()
+ self.store.abortTransaction()
Index: chandler/repository/persistence/Repository.py
diff -u chandler/repository/persistence/Repository.py:1.70 chandler/repository/persistence/Repository.py:1.71
--- chandler/repository/persistence/Repository.py:1.70 Thu Apr 1 11:51:40 2004
+++ chandler/repository/persistence/Repository.py Thu May 13 23:34:23 2004
@@ -1,6 +1,6 @@
-__revision__ = "$Revision: 1.70 $"
-__date__ = "$Date: 2004/04/01 19:51:40 $"
+__revision__ = "$Revision: 1.71 $"
+__date__ = "$Date: 2004/05/14 06:34:23 $"
__copyright__ = "Copyright (c) 2002 Open Source Applications Foundation"
__license__ = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
@@ -30,6 +30,12 @@
def getItem(self):
return self.args[0]
+class NoSuchItemError(RepositoryError):
+ "No such item %s, version %d"
+
+ def __str__(self):
+ return self.__doc__ % self.args
+
class Repository(object):
"""An abstract item repository.
@@ -54,10 +60,6 @@
self._init()
- def serverOpen(self):
-
- raise NotImplementedError, "Repository.serverOpen"
-
def delete(self):
raise NotImplementedError, "Repository.delete"
@@ -501,6 +503,10 @@
return path
+ def _getStore(self):
+
+ return self.repository.store
+
def logItem(self, item):
if not self.repository.isOpen():
@@ -603,6 +609,7 @@
logger = property(getLogger)
debug = property(isDebug)
+ store = property(_getStore)
OPEN = 0x1
LOADING = 0x2
@@ -627,15 +634,24 @@
def loadItem(self, version, uuid):
raise NotImplementedError, "Store.loadItem"
+ def serveItem(self, version, uuid):
+ raise NotImplementedError, "Store.serveItem"
+
def loadChild(self, version, uuid, name):
raise NotImplementedError, "Store.loadChild"
+ def serveChild(self, version, uuid, name):
+ raise NotImplementedError, "Store.serveChild"
+
def loadRoots(self, version):
raise NotImplementedError, "Store.loadRoots"
def loadRef(self, version, uItem, uuid, key):
raise NotImplementedError, "Store.loadRef"
+ def loadRefs(self, version, uItem, uuid, firstKey):
+ raise NotImplementedError, "Store.loadRefs"
+
def queryItems(self, version, query):
raise NotImplementedError, "Store.queryItems"
@@ -674,8 +690,8 @@
super(OnDemandRepositoryView, self).__init__(repository)
- self._hooks = None
self.version = repository.store.getVersion()
+ self._hooks = None
self._notRoots = {}
def _loadDoc(self, doc):
@@ -836,7 +852,7 @@
self.logger.info('pruning %d items', count)
for i in xrange(count):
registry[heapq.heappop(heap)[1]]._unloadItem()
-
+
class RepositoryNotifications(dict):
Index: chandler/repository/persistence/XMLRepository.py
diff -u chandler/repository/persistence/XMLRepository.py:1.70 chandler/repository/persistence/XMLRepository.py:1.71
--- chandler/repository/persistence/XMLRepository.py:1.70 Fri Apr 9 12:22:37 2004
+++ chandler/repository/persistence/XMLRepository.py Thu May 13 23:34:23 2004
@@ -1,33 +1,36 @@
-__revision__ = "$Revision: 1.70 $"
-__date__ = "$Date: 2004/04/09 19:22:37 $"
+__revision__ = "$Revision: 1.71 $"
+__date__ = "$Date: 2004/05/14 06:34:23 $"
__copyright__ = "Copyright (c) 2002 Open Source Applications Foundation"
__license__ = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
-import os, os.path, re, libxml2
+import os, os.path, re, libxml2, cStringIO
from datetime import datetime
from threading import currentThread
+from struct import pack
+from repository.item.Item import Item
from repository.util.UUID import UUID
from repository.util.ThreadLocal import ThreadLocal
+from repository.util.SAX import XMLGenerator
from repository.persistence.Repository import Repository, RepositoryError
from repository.persistence.Repository import OnDemandRepository, Store
-from repository.persistence.XMLRepositoryView import XMLRepositoryLocalView
-from repository.persistence.XMLRepositoryView import XMLRepositoryClientView
+from repository.persistence.XMLRepositoryView import XMLRepositoryView
from repository.persistence.DBContainer import DBContainer, RefContainer
from repository.persistence.DBContainer import VerContainer, HistContainer
from repository.persistence.DBContainer import NamesContainer
from repository.persistence.FileContainer import FileContainer, BlockContainer
from repository.persistence.FileContainer import IndexContainer
+from repository.remote.CloudFilter import CloudFilter
from bsddb.db import DBEnv, DB, DBError
-from bsddb.db import DB_CREATE, DB_BTREE, DB_THREAD
+from bsddb.db import DB_CREATE, DB_BTREE, DB_THREAD, DB_LOCK_WRITE
from bsddb.db import DB_RECOVER, DB_RECOVER_FATAL, DB_LOCK_MINLOCKS
from bsddb.db import DB_INIT_MPOOL, DB_INIT_LOCK, DB_INIT_TXN, DB_DIRTY_READ
from bsddb.db import DBRunRecoveryError, DBNoSuchFileError, DBNotFoundError
from bsddb.db import DBLockDeadlockError
-from dbxml import XmlContainer, XmlValue
+from dbxml import XmlContainer, XmlDocument, XmlValue
from dbxml import XmlQueryContext, XmlUpdateContext
@@ -63,9 +66,13 @@
self._env = self._createEnv()
self._env.open(self.dbHome, DB_CREATE | self.OPEN_FLAGS, 0)
- self.store = XMLStore(self)
+ self.store = self._createStore()
self.store.open(True)
+ def _createStore(self):
+
+ return XMLStore(self)
+
def _createEnv(self):
env = DBEnv()
@@ -102,7 +109,7 @@
else:
self._env.open(self.dbHome, self.OPEN_FLAGS, 0)
- self.store = XMLStore(self)
+ self.store = self._createStore()
self.store.open(False)
except DBNoSuchFileError:
@@ -123,16 +130,9 @@
self._env = None
self._status &= ~self.OPEN
- def serverOpen(self):
-
- if not self.isOpen():
- raise RepositoryError, "Repository is not open"
-
- return (self.store, XMLRepositoryClientView)
-
def _createView(self):
- return XMLRepositoryLocalView(self)
+ return XMLRepositoryView(self)
OPEN_FLAGS = DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN | DB_THREAD
@@ -177,7 +177,7 @@
store = self.store
txnStarted = False
try:
- txnStarted = store._startTransaction()
+ txnStarted = store.startTransaction()
docId = store._versions.getDocId(uuid, version)
# None -> not found, 0 -> deleted
@@ -186,7 +186,7 @@
finally:
if txnStarted:
- store._abortTransaction()
+ store.abortTransaction()
return None
@@ -205,21 +205,20 @@
ctx.setEvaluationType(XmlQueryContext.Lazy)
ctx.setVariableValue("uuid", XmlValue(Repository.itsUUID.str64()))
ctx.setVariableValue("version", XmlValue(float(version)))
- nameExp = re.compile("<name>(.*)</name>")
roots = {}
store = self.store
view = store.repository.view
txnStarted = False
try:
- txnStarted = store._startTransaction()
+ txnStarted = store.startTransaction()
if self.version in ('1.2.0', '1.2.1'):
for value in self._xml.queryWithXPath(store.txn,
"/item[container=$uuid and number(@version)<=$version]",
ctx, DB_DIRTY_READ):
doc = value.asDocument()
xml = doc.getContent()
- match = nameExp.match(xml, xml.index("<name>"))
+ match = self.nameExp.match(xml, xml.index("<name>"))
name = match.group(1)
if not name in view._roots:
@@ -231,7 +230,7 @@
finally:
if txnStarted:
- store._abortTransaction()
+ store.abortTransaction()
value = XmlValue()
for name, (ver, doc) in roots.iteritems():
@@ -251,7 +250,7 @@
docs = {}
try:
- txnStarted = store._startTransaction()
+ txnStarted = store.startTransaction()
for value in self._xml.queryWithXPath(store.txn,
query, store.ctx,
DB_DIRTY_READ):
@@ -265,7 +264,7 @@
finally:
if txnStarted:
- store._abortTransaction()
+ store.abortTransaction()
results = []
for uuid, (doc, ver) in docs.iteritems():
@@ -315,6 +314,8 @@
return long(xml[index:xml.index('"', index)])
+ nameExp = re.compile("<name>(.*)</name>")
+
class XMLStore(Store):
@@ -328,7 +329,7 @@
txnStarted = False
try:
- txnStarted = self._startTransaction()
+ txnStarted = self.startTransaction()
txn = self.txn
self._data = XMLContainer(self, "__data__", txn, create)
@@ -342,7 +343,7 @@
self._index = IndexContainer(self, "__index__", txn, create)
finally:
if txnStarted:
- self._commitTransaction()
+ self.commitTransaction()
def close(self):
@@ -394,9 +395,33 @@
def loadRef(self, version, uItem, uuid, key):
- return self._refs.loadRef(version, key, "".join((uItem._uuid,
- uuid._uuid,
- key._uuid)))
+ buffer = self._refs.prepareKey(uItem, uuid)
+ try:
+ return self._refs.loadRef(buffer, version, key)
+ finally:
+ buffer.close()
+
+ def loadRefs(self, version, uItem, uuid, firstKey):
+
+ refs = []
+
+ buffer = self._refs.prepareKey(uItem, uuid)
+ txnStarted = False
+ try:
+ txnStarted = self.startTransaction()
+ key = firstKey
+ while key is not None:
+ ref = self._refs.loadRef(buffer, version, key)
+ assert ref is not None
+
+ refs.append(ref)
+ key = ref[3]
+ finally:
+ if txnStarted:
+ self.abortTransaction()
+ buffer.close()
+
+ return refs
def queryItems(self, version, query):
@@ -426,7 +451,7 @@
return self._versions.getVersion()
- def _startTransaction(self):
+ def startTransaction(self):
if self.txn is None:
self.txn = self.repository._env.txn_begin(None, DB_DIRTY_READ)
@@ -434,7 +459,7 @@
return False
- def _commitTransaction(self):
+ def commitTransaction(self):
if self.txn is not None:
self.txn.commit()
@@ -443,7 +468,7 @@
return False
- def _abortTransaction(self):
+ def abortTransaction(self):
if self.txn is not None:
self.txn.abort()
@@ -452,6 +477,10 @@
return False
+ def lobName(self, uuid, version):
+
+ return pack('>16sl', uuid._uuid, ~version)
+
def _getTxn(self):
try:
@@ -493,7 +522,95 @@
return updateCtx
+ def _getLockId(self):
+
+ try:
+ return self._threaded.lockId
+ except AttributeError:
+ lockId = self.repository._env.lock_id()
+ self._threaded.lockId = lockId
+
+ return lockId
+
+ def acquireLock(self):
+
+ repository = self.repository
+ return repository._env.lock_get(self.lockId, repository.itsUUID._uuid,
+ DB_LOCK_WRITE)
+
+ def releaseLock(self, lock):
+
+ self.repository._env.lock_put(lock)
+ return None
+
+ def saveItem(self, xml, uuid, version, currPN, origPN, status):
+
+ doc = XmlDocument()
+ doc.setContent(xml)
+ if status & Item.DELETED:
+ doc.setMetaData('', '', 'deleted', XmlValue('True'))
+
+ try:
+ docId = self._data.putDocument(doc)
+ except:
+ self.repository.logger.exception("putDocument failed, xml is: %s",
+ xml)
+ raise
+
+ if status & Item.DELETED:
+ parent, name = origPN
+ self._versions.setDocVersion(uuid, version, 0)
+ self._history.writeVersion(uuid, version, 0, status, parent)
+ self._names.writeName(parent, name, version, None)
+
+ else:
+ self._versions.setDocVersion(uuid, version, docId)
+ self._history.writeVersion(uuid, version, docId, status)
+
+ if origPN is not None:
+ parent, name = origPN
+ self._names.writeName(parent, name, version, None)
+
+ parent, name = currPN
+ self._names.writeName(parent, name, version, uuid)
+
+ def serveItem(self, version, uuid):
+
+ if version == 0:
+ version = self._versions.getVersion()
+
+ doc = self.loadItem(version, uuid)
+ if doc is None:
+ return None
+
+ xml = doc.getContent()
+ out = cStringIO.StringIO()
+ generator = XMLGenerator(out)
+
+ try:
+ generator.startElement('items',
+ { 'version': str(version) })
+ filter = CloudFilter(None, self, uuid, version, generator)
+ filter.parse(xml, {})
+ generator.endElement('items')
+
+ return out.getvalue()
+ finally:
+ out.close()
+
+ def serveChild(self, version, uuid, name):
+
+ if version == 0:
+ version = self._versions.getVersion()
+
+ uuid = self._names.readName(uuid, name, version)
+ if uuid is None:
+ return None
+
+ return self.serveItem(version, uuid)
+
env = property(_getEnv)
ctx = property(_getCtx)
updateCtx = property(_getUpdateCtx)
txn = property(_getTxn, _setTxn)
+ lockId = property(_getLockId)
Index: chandler/repository/schema/Types.py
diff -u chandler/repository/schema/Types.py:1.57 chandler/repository/schema/Types.py:1.58
--- chandler/repository/schema/Types.py:1.57 Fri Mar 26 11:25:49 2004
+++ chandler/repository/schema/Types.py Thu May 13 23:34:26 2004
@@ -1,6 +1,6 @@
-__revision__ = "$Revision: 1.57 $"
-__date__ = "$Date: 2004/03/26 19:25:49 $"
+__revision__ = "$Revision: 1.58 $"
+__date__ = "$Date: 2004/05/14 06:34:26 $"
__copyright__ = "Copyright (c) 2002 Open Source Applications Foundation"
__license__ = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
@@ -852,7 +852,7 @@
def textEnd(self, itemHandler, attrs):
- itemHandler.value._textEnd(itemHandler.data, attrs)
+ itemHandler.value.load(itemHandler.data, attrs)
itemHandler.tagCounts[-1] -= 1
def typeXML(self, value, generator, withSchema):
@@ -886,7 +886,7 @@
def binaryEnd(self, itemHandler, attrs):
- itemHandler.value._binaryEnd(itemHandler.data, attrs)
+ itemHandler.value.load(itemHandler.data, attrs)
itemHandler.tagCounts[-1] -= 1
def typeXML(self, value, generator, withSchema):
Index: chandler/repository/util/SAX.py
diff -u chandler/repository/util/SAX.py:1.3 chandler/repository/util/SAX.py:1.4
--- chandler/repository/util/SAX.py:1.3 Thu Apr 1 11:51:41 2004
+++ chandler/repository/util/SAX.py Thu May 13 23:34:27 2004
@@ -1,6 +1,6 @@
-__revision__ = "$Revision: 1.3 $"
-__date__ = "$Date: 2004/04/01 19:51:41 $"
+__revision__ = "$Revision: 1.4 $"
+__date__ = "$Date: 2004/05/14 06:34:27 $"
__copyright__ = "Copyright (c) 2002 Open Source Applications Foundation"
__license__ = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
@@ -55,6 +55,10 @@
self.out = out
self.encoding = encoding
+ def getOutputStream(self):
+
+ return self.out
+
def write(self, data):
self.out.write(data)
@@ -91,30 +95,97 @@
def characters(self, data):
- if isinstance(data, unicode):
- data = data.encode(self.encoding)
+ if data:
+ if isinstance(data, unicode):
+ data = data.encode(self.encoding)
- self.out.write(escape(None, data))
+ self.out.write(escape(None, data))
- def cdataSection(self, data):
+ def cdataSection(self, data, start=True, end=True):
- if isinstance(data, unicode):
+ if data and isinstance(data, unicode):
data = data.encode(self.encoding)
- self.out.write('<![CDATA[')
- self.out.write(data)
- self.out.write(']]>')
+ if start:
+ self.out.write('<![CDATA[')
+ if data:
+ self.out.write(data)
+ if end:
+ self.out.write(']]>')
+
+
+class XMLPrettyGenerator(XMLGenerator):
+
+ def __init__(self, generator):
+
+ self.generator = generator
+ self._indent = ' '
+ self._indents = 0
+ self._nl = False
+
+ super(XMLPrettyGenerator, self).__init__(None)
+
+ def getOutputStream(self):
+
+ return self.generator.getOutputStream()
+
+ def write(self, data):
+
+ self.generator.write(data)
+
+ def startDocument(self):
+
+ self._indents = 0
+ self.generator.startDocument()
+
+ def endDocument(self):
+
+ self.generator.endDocument()
+
+ def startElement(self, tag, attrs):
+
+ self.write('\n')
+ for i in xrange(self._indents):
+ self.write(self._indent)
+ self.generator.startElement(tag, attrs)
+ self._indents += 1
+
+ def endElement(self, tag):
+
+ self._indents -= 1
+ if self._nl:
+ self.write('\n')
+ for i in xrange(self._indents):
+ self.write(self._indent)
+ self.generator.endElement(tag)
+ self._nl = True
+
+ def characters(self, data):
+
+ self.generator.characters(data)
+ self._nl = False
+
+ def cdataSection(self, data, start=True, end=True):
+
+ self.generator.cdataSection(data, start, end)
+ self._nl = False
class XMLFilter(ContentHandler):
def __init__(self, generator, *tags):
+ ContentHandler.__init__(self)
+
self.generator = generator
self.tags = tags
self.foundTag = 0
self.cdata = False
+ def getGenerator(self):
+
+ return self.generator
+
def output(self):
raise NotImplementedError, 'XMLFilter.output'
@@ -122,6 +193,8 @@
def parse(self, xml):
createPushParser(self, xml, len(xml), 'filter').parseChunk('', 0, 1)
+ if self.errorOccurred():
+ raise self.saxError()
def endDocument(self):
More information about the Commits
mailing list