[Commits] (vajda) - added --recover/-R to Chandler startup flags

commits at osafoundation.org commits at osafoundation.org
Tue Feb 15 16:29:23 PST 2005


Commit by: vajda
Modified files:
chandler/Chandler.py 1.61 1.62
chandler/application/Application.py 1.299 1.300
chandler/repository/item/Item.py 1.195 1.196
chandler/repository/persistence/DBContainer.py 1.35 1.36
chandler/repository/persistence/DBRepository.py 1.5 1.6
chandler/repository/schema/Types.py 1.76 1.77
chandler/repository/util/LinkedMap.py 1.25 1.26
chandler/repository/util/Lob.py 1.10 1.11
chandler/repository/util/Streams.py 1.19 1.20

Log message:
   - added --recover/-R to Chandler startup flags
   - changed config logic to write DB_CONFIG on create
   - added support for auto-detection of DB_RECOVER need
   - repository no longer always opened with DB_RECOVER
   - turned on DB_DSYNC_LOG on OS X
   - added optional 'replace' argument to Lob and Stream unicode APIs
   - added makeCollection APIs on collection schema types
   - fixed bug in an renaming anonymous item


ViewCVS links:
http://cvs.osafoundation.org/index.cgi/chandler/Chandler.py.diff?r1=text&tr1=1.61&r2=text&tr2=1.62
http://cvs.osafoundation.org/index.cgi/chandler/application/Application.py.diff?r1=text&tr1=1.299&r2=text&tr2=1.300
http://cvs.osafoundation.org/index.cgi/chandler/repository/item/Item.py.diff?r1=text&tr1=1.195&r2=text&tr2=1.196
http://cvs.osafoundation.org/index.cgi/chandler/repository/persistence/DBContainer.py.diff?r1=text&tr1=1.35&r2=text&tr2=1.36
http://cvs.osafoundation.org/index.cgi/chandler/repository/persistence/DBRepository.py.diff?r1=text&tr1=1.5&r2=text&tr2=1.6
http://cvs.osafoundation.org/index.cgi/chandler/repository/schema/Types.py.diff?r1=text&tr1=1.76&r2=text&tr2=1.77
http://cvs.osafoundation.org/index.cgi/chandler/repository/util/LinkedMap.py.diff?r1=text&tr1=1.25&r2=text&tr2=1.26
http://cvs.osafoundation.org/index.cgi/chandler/repository/util/Lob.py.diff?r1=text&tr1=1.10&r2=text&tr2=1.11
http://cvs.osafoundation.org/index.cgi/chandler/repository/util/Streams.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20

Index: chandler/repository/persistence/DBContainer.py
diff -u chandler/repository/persistence/DBContainer.py:1.35 chandler/repository/persistence/DBContainer.py:1.36
--- chandler/repository/persistence/DBContainer.py:1.35	Tue Feb  8 15:45:14 2005
+++ chandler/repository/persistence/DBContainer.py	Tue Feb 15 16:29:20 2005
@@ -1,6 +1,6 @@
 
-__revision__  = "$Revision: 1.35 $"
-__date__      = "$Date: 2005/02/08 23:45:14 $"
+__revision__  = "$Revision: 1.36 $"
+__date__      = "$Date: 2005/02/16 00:29:20 $"
 __copyright__ = "Copyright (c) 2002 Open Source Applications Foundation"
 __license__   = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
 
@@ -1127,7 +1127,7 @@
         if value is None:
             return None
 
-        versionId, version, format = unpack('>16sLL', value)
+        versionId, version, format = unpack('>16sll', value)
 
         return UUID(versionId), version, format
         
@@ -1140,7 +1140,7 @@
         if value is None:
             return None
 
-        return unpack('>L', value[16:20])[0]
+        return unpack('>l', value[16:20])[0]
         
     def setVersion(self, version, uuid=None):
         
@@ -1152,12 +1152,12 @@
         else:
             versionId, format = UUID(), ValueContainer.FORMAT_VERSION
 
-        self.put(uuid._uuid, pack('>16sLL', versionId._uuid, version, format))
+        self.put(uuid._uuid, pack('>16sll', versionId._uuid, version, format))
 
     def setVersionId(self, versionId, uuid):
 
         versionId, version, format = self.getVersionInfo(uuid)
-        self.put(uuid._uuid, pack('>16sLL', versionId._uuid, version, format))
+        self.put(uuid._uuid, pack('>16sll', versionId._uuid, version, format))
 
 
 class HashTuple(tuple):

Index: chandler/application/Application.py
diff -u chandler/application/Application.py:1.299 chandler/application/Application.py:1.300
--- chandler/application/Application.py:1.299	Mon Feb  7 14:53:54 2005
+++ chandler/application/Application.py	Tue Feb 15 16:29:19 2005
@@ -1,5 +1,5 @@
-__version__ = "$Revision: 1.299 $"
-__date__ = "$Date: 2005/02/07 22:53:54 $"
+__version__ = "$Revision: 1.300 $"
+__date__ = "$Date: 2005/02/16 00:29:19 $"
 __copyright__ = "Copyright (c) 2003-2004 Open Source Applications Foundation"
 __license__ = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
 
@@ -207,12 +207,13 @@
             path = os.sep.join([Globals.options.profileDir, '__repository__'])
         else:
             path = '__repository__'
-        
-        kwds = { 'stderr': Globals.options.stderr,
-                 'ramdb': Globals.options.ramdb,
+
+        options = Globals.options
+        kwds = { 'stderr': options.stderr,
+                 'ramdb': options.ramdb,
                  'create': True,
-                 'recover': True,
-                 'exclusive': Globals.options.exclusive,
+                 'recover': options.recover,
+                 'exclusive': options.exclusive,
                  'refcounted': True }
                  
         if Globals.options.repo:

Index: chandler/repository/item/Item.py
diff -u chandler/repository/item/Item.py:1.195 chandler/repository/item/Item.py:1.196
--- chandler/repository/item/Item.py:1.195	Fri Feb  4 13:39:34 2005
+++ chandler/repository/item/Item.py	Tue Feb 15 16:29:19 2005
@@ -1,6 +1,6 @@
 
-__revision__  = "$Revision: 1.195 $"
-__date__      = "$Date: 2005/02/04 21:39:34 $"
+__revision__  = "$Revision: 1.196 $"
+__date__      = "$Date: 2005/02/16 00:29:19 $"
 __copyright__ = "Copyright (c) 2003-2004 Open Source Applications Foundation"
 __license__   = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
 
@@ -74,12 +74,12 @@
         if kind is not None:
             kind._setupClass(cls)
 
-        if parent._isRepository():
+        if not parent._isItem():
             if kind is not None:
                 parent = kind.getAttributeValue('defaultParent', default=parent)
                 if parent is None:
                     raise NoSuchDefaultParentError, (self, kind)
-            if name is None and parent._isRepository():
+            if name is None and not parent._isItem():
                 raise ValueError, 'repository root cannot be anonymous'
 
         self._setParent(parent)

Index: chandler/repository/schema/Types.py
diff -u chandler/repository/schema/Types.py:1.76 chandler/repository/schema/Types.py:1.77
--- chandler/repository/schema/Types.py:1.76	Tue Jan 25 12:40:46 2005
+++ chandler/repository/schema/Types.py	Tue Feb 15 16:29:20 2005
@@ -1,6 +1,6 @@
 
-__revision__  = "$Revision: 1.76 $"
-__date__      = "$Date: 2005/01/25 20:40:46 $"
+__revision__  = "$Revision: 1.77 $"
+__date__      = "$Date: 2005/02/16 00:29:20 $"
 __copyright__ = "Copyright (c) 2003-2004 Open Source Applications Foundation"
 __license__   = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
 
@@ -1006,6 +1006,10 @@
                                   generator, withSchema)
         generator.endElement('values')
 
+    def makeString(self, value):
+
+        return ",".join(["%s:%s" %(k, v) for k, v in value.iteritems()])
+
     def makeValue(self, data):
         """
         Make a dict of string key/value pairs from comma separated pairs.
@@ -1022,9 +1026,18 @@
 
         return result
 
-    def makeString(self, value):
+    def makeCollection(self, values):
 
-        return ",".join(["%s:%s" %(k, v) for k, v in value.iteritems()])
+        result = {}
+
+        count = len(values)
+        if count % 2:
+            raise ValueError, 'keys/values list is not of even length'
+
+        for i in xrange(0, count, 2):
+            result[values[i]] = values[i + 1]
+
+        return result
 
     def _empty(self):
 
@@ -1060,6 +1073,10 @@
                                   generator, withSchema)
         generator.endElement('values')
 
+    def makeString(self, value):
+
+        return ",".join([str(v) for v in value])
+
     def makeValue(self, data):
         """
         Make a list of strings from comma separated strings.
@@ -1073,9 +1090,9 @@
         else:
             return []
 
-    def makeString(self, value):
+    def makeCollection(self, values):
 
-        return ",".join([str(v) for v in value])
+        return list(values)
 
     def _empty(self):
 
@@ -1115,6 +1132,10 @@
                                   generator, withSchema)
         generator.endElement('values')
 
+    def makeString(self, value):
+
+        return ",".join([str(v) for v in value])
+
     def makeValue(self, data):
         """
         Make a tuple of strings from comma separated strings.
@@ -1128,9 +1149,9 @@
         else:
             return ()
 
-    def makeString(self, value):
+    def makeCollection(self, values):
 
-        return ",".join([str(v) for v in value])
+        return tuple(values)
 
     def _empty(self):
 
@@ -1171,7 +1192,7 @@
 
     def makeValue(self, data,
                   encoding=None, mimetype='text/plain', compression='bz2',
-                  encryption=None, key=None, indexed=False):
+                  encryption=None, key=None, indexed=False, replace=False):
 
         if data and not encoding and type(data) is unicode:
             encoding = 'utf-8'
@@ -1181,7 +1202,7 @@
 
         if data:
             if encoding:
-                out = lob.getWriter(compression, encryption, key)
+                out = lob.getWriter(compression, encryption, key, replace)
             else:
                 out = lob.getOutputStream(compression, encryption, key)
             out.write(data)

Index: chandler/repository/util/LinkedMap.py
diff -u chandler/repository/util/LinkedMap.py:1.25 chandler/repository/util/LinkedMap.py:1.26
--- chandler/repository/util/LinkedMap.py:1.25	Tue Jan 25 12:40:49 2005
+++ chandler/repository/util/LinkedMap.py	Tue Feb 15 16:29:21 2005
@@ -1,6 +1,6 @@
 
-__revision__  = "$Revision: 1.25 $"
-__date__      = "$Date: 2005/01/25 20:40:49 $"
+__revision__  = "$Revision: 1.26 $"
+__date__      = "$Date: 2005/02/16 00:29:21 $"
 __copyright__ = "Copyright (c) 2003-2004 Open Source Applications Foundation"
 __license__   = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
 
@@ -339,7 +339,10 @@
                     pass
 
             link._alias = alias
-            self._aliases[alias] = key
+            if self._aliases is None:
+                self._aliases = {alias: key}
+            else:
+                self._aliases[alias] = key
 
     def firstKey(self):
         "Return the first key of this mapping."

Index: chandler/Chandler.py
diff -u chandler/Chandler.py:1.61 chandler/Chandler.py:1.62
--- chandler/Chandler.py:1.61	Mon Feb  7 19:47:48 2005
+++ chandler/Chandler.py	Tue Feb 15 16:29:19 2005
@@ -1,5 +1,5 @@
-__version__ = "$Revision: 1.61 $"
-__date__ = "$Date: 2005/02/08 03:47:48 $"
+__version__ = "$Revision: 1.62 $"
+__date__ = "$Date: 2005/02/16 00:29:19 $"
 __copyright__ = "Copyright (c) 2003-2005 Open Source Applications Foundation"
 __license__ = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
 
@@ -70,6 +70,7 @@
                      'ramdb':      ('-d', '--ramdb',      'b', False, None, ''),
                      'exclusive':  ('-x', '--exclusive',  'b', False, None, 'open repository exclusive'),
                      'repo':       ('-r', '--repo',       's', None,  None, 'repository to copy during startup'),
+                     'recover':    ('-R', '--recover',    'b', False, None, 'open repository with recovery'),
                      'nocatch':    ('-n', '--nocatch',    'b', False, None, ''),
                      'wing':       ('-w', '--wing',       'b', False, None, ''),
                      'komodo':     ('-k', '--komodo',     'b', False, None, ''),

Index: chandler/repository/persistence/DBRepository.py
diff -u chandler/repository/persistence/DBRepository.py:1.5 chandler/repository/persistence/DBRepository.py:1.6
--- chandler/repository/persistence/DBRepository.py:1.5	Wed Jan 26 12:18:00 2005
+++ chandler/repository/persistence/DBRepository.py	Tue Feb 15 16:29:20 2005
@@ -1,6 +1,6 @@
 
-__revision__  = "$Revision: 1.5 $"
-__date__      = "$Date: 2005/01/26 20:18:00 $"
+__revision__  = "$Revision: 1.6 $"
+__date__      = "$Date: 2005/02/16 00:29:20 $"
 __copyright__ = "Copyright (c) 2003-2004 Open Source Applications Foundation"
 __license__   = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
 
@@ -29,13 +29,16 @@
 from repository.remote.CloudFilter import CloudFilter
 
 from bsddb.db import DBEnv, DB, DBError
-from bsddb.db import DB_CREATE, DB_BTREE, DB_THREAD
+from bsddb.db import DB_CREATE, DB_BTREE, DB_THREAD, DB_REGION_INIT
 from bsddb.db import DB_LOCK_WRITE
 from bsddb.db import DB_RECOVER, DB_RECOVER_FATAL, DB_PRIVATE, DB_LOCK_MINLOCKS
-from bsddb.db import DB_INIT_MPOOL, DB_INIT_LOCK, DB_INIT_TXN
+from bsddb.db import DB_INIT_MPOOL, DB_INIT_LOCK, DB_INIT_LOG, DB_INIT_TXN
 from bsddb.db import DBRunRecoveryError, DBNoSuchFileError, DBNotFoundError
 from bsddb.db import DBLockDeadlockError
 
+# missing from python interface at the moment
+DB_DSYNC_LOG = 0x00008000
+
 
 class DBRepository(OnDemandRepository):
     """
@@ -48,16 +51,42 @@
         """
         
         super(DBRepository, self).__init__(dbHome)
+
         self._openLock = None
+        self._openFile = None
+        if dbHome is not None:
+            self._openDir = os.path.join(self.dbHome, '__open')
+        else:
+            self._openDir = None
         self._exclusiveLock = None
         self._env = None
-        
+
+    def _touchOpenFile(self):
+
+        if self._openFile is None:
+            self._openFile = os.path.join(self._openDir, UUID().str64())
+
+        if not os.path.exists(self._openDir):
+            os.mkdir(self._openDir)
+            
+        file(self._openFile, "w+").close()
+
+    def _clearOpenDir(self):
+
+        if os.path.exists(self._openDir):
+            for name in os.listdir(self._openDir):
+                path = os.path.join(self._openDir, name)
+                if not os.path.isdir(path):
+                    os.remove(path)
+            
     def create(self, **kwds):
 
         if not self.isOpen():
             super(DBRepository, self).create(**kwds)
             self._create(**kwds)
             self._status |= self.OPEN
+            if not kwds.get('ramdb', False):
+                self._touchOpenFile()
 
     def _create(self, **kwds):
 
@@ -75,7 +104,7 @@
 
         if kwds.get('ramdb', False):
             flags = DB_INIT_MPOOL | DB_PRIVATE | DB_THREAD
-            self._env = self._createEnv()
+            self._env = self._createEnv(True, kwds)
             self._env.open(self.dbHome, DB_CREATE | flags, 0)
             
         else:
@@ -87,7 +116,7 @@
                 self.delete()
 
             self._lockOpen()
-            self._env = self._createEnv()
+            self._env = self._createEnv(True, kwds)
             self._env.open(self.dbHome, DB_CREATE | self.OPEN_FLAGS, 0)
 
         self.store = self._createStore()
@@ -118,38 +147,65 @@
             lock.close(self._openLock)
             self._openLock = None
 
-    def _createEnv(self):
+    def _createEnv(self, create, kwds):
 
         env = DBEnv()
-        env.set_lk_detect(DB_LOCK_MINLOCKS)
-        env.set_lk_max_locks(32767)
-        env.set_lk_max_objects(32767)
-
-        #
-        # create a 64Mb cache on Windows and Linux
-        #
+
+        ramdb = kwds.get('ramdb', False)
+        locks = 32767
+        cache = 0x4000000
+        logbs = 0x2000000
+        
+        if create and not ramdb:
+            db_config = file(os.path.join(self.dbHome, 'DB_CONFIG'), 'w+b')
+
+        if create or ramdb:
+            env.set_lk_detect(DB_LOCK_MINLOCKS)
+            env.set_lk_max_locks(locks)
+            env.set_lk_max_objects(locks)
+        if create and not ramdb:
+            db_config.write("set_lk_detect DB_LOCK_MINLOCKS\n")
+            db_config.write("set_lk_max_locks %d\n" %(locks))
+            db_config.write("set_lk_max_objects %d\n" %(locks))
 
         if os.name == 'nt':
-            env.set_cachesize(0, 0x4000000, 1)
+            if create or ramdb:
+                env.set_cachesize(0, cache, 1)
+            if create and not ramdb:
+                db_config.write("set_cachesize 0 %d 1\n" %(cache))
 
         elif os.name == 'posix':
             from commands import getstatusoutput
 
-            if getstatusoutput('uname') == (0, 'Linux'):
-                env.set_cachesize(0, 0x4000000, 1)
+            status, osname = getstatusoutput('uname')
+            if status == 0:
+
+                if osname == 'Linux':
+                    if create or ramdb:
+                        env.set_cachesize(0, cache, 1)
+                    if create and not ramdb:
+                        db_config.write("set_cachesize 0 %d 1\n" %(cache))
+
+                elif osname == 'Darwin':
+                    if create or ramdb:
+                        env.set_flags(DB_DSYNC_LOG, 1)
+                    if create and not ramdb:
+                        db_config.write("set_flags DB_DSYNC_LOG\n")
+
+        if create and not ramdb:
+            db_config.close()
 
         return env
 
     def delete(self):
 
-        def purge(arg, path, names):
-            for f in names:
-                if f.startswith('__') or f.startswith('log.'):
-                    f = os.path.join(path, f)
-                    if not os.path.isdir(f):
-                        os.remove(f)
-                        
-        os.path.walk(self.dbHome, purge, None)
+        for name in os.listdir(self.dbHome):
+            if name.startswith('__') or name.startswith('log.'):
+                path = os.path.join(self.dbHome, name)
+                if not os.path.isdir(path):
+                    os.remove(path)
+
+        self._clearOpenDir()
 
     def open(self, **kwds):
 
@@ -165,16 +221,22 @@
                 import shutil
                 for f in os.listdir(fromPath):
                     if f.startswith('__') or f.startswith('log.'):
-                        shutil.copy2(os.path.join(fromPath,f), self.dbHome)
+                        path = os.path.join(fromPath, f)
+                        if not os.path.isdir(path):
+                            shutil.copy2(path, self.dbHome)
 
             super(DBRepository, self).open(**kwds)
 
             self._lockOpen()
-            self._env = self._createEnv()
+            self._env = self._createEnv(False, kwds)
 
             recover = kwds.get('recover', False)
             exclusive = kwds.get('exclusive', False)
 
+            if not recover:
+                if os.path.exists(self._openDir) and os.listdir(self._openDir):
+                    recover = True
+
             try:
                 if recover or exclusive:
                     try:
@@ -196,8 +258,12 @@
                             after = datetime.now()
                             self.logger.info('opened db with recovery in %s',
                                              after - before)
+                            self._clearOpenDir()
                         else:
+                            before = datetime.now()
                             self._env.open(self.dbHome, self.OPEN_FLAGS, 0)
+                            after = datetime.now()
+                            self.logger.info('opened db in %s', after - before)
 
                     finally:
                         if locked:
@@ -206,7 +272,10 @@
                             else:
                                 lock.lock(fd, lock.LOCK_UN | lock.LOCK_SH)
                 else:
+                    before = datetime.now()
                     self._env.open(self.dbHome, self.OPEN_FLAGS, 0)
+                    after = datetime.now()
+                    self.logger.info('opened db in %s', after - before)
 
                 self.store = self._createStore()
                 kwds['create'] = False
@@ -216,10 +285,13 @@
                 kwds['create'] = recover
                 if kwds.get('create', False):
                     self._create(**kwds)
+                elif not os.path.exists(self.dbHome):
+                    self._create(**kwds)
                 else:
                     raise
 
             self._status |= self.OPEN
+            self._touchOpenFile()
 
     def close(self):
 
@@ -231,6 +303,9 @@
             self._env = None
             self._lockClose()
             self._status &= ~self.OPEN
+            if self._openFile is not None:
+                os.remove(self._openFile)
+                self._openFile = None
 
     def createView(self, name=None):
 

Index: chandler/repository/util/Lob.py
diff -u chandler/repository/util/Lob.py:1.10 chandler/repository/util/Lob.py:1.11
--- chandler/repository/util/Lob.py:1.10	Thu Jan 13 17:13:43 2005
+++ chandler/repository/util/Lob.py	Tue Feb 15 16:29:21 2005
@@ -1,6 +1,6 @@
 
-__revision__  = "$Revision: 1.10 $"
-__date__      = "$Date: 2005/01/14 01:13:43 $"
+__revision__  = "$Revision: 1.11 $"
+__date__      = "$Date: 2005/02/16 00:29:21 $"
 __copyright__ = "Copyright (c) 2003-2004 Open Source Applications Foundation"
 __license__   = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
 
@@ -92,29 +92,28 @@
         return self._data
 
     def getWriter(self, compression='bz2', encryption=None, key=None,
-                  append=False):
+                  append=False, replace=False):
 
         return OutputStreamWriter(self.getOutputStream(compression, encryption,
                                                        key, append),
-                                  self.encoding)
+                                  self.encoding, replace)
 
-    def getReader(self, key=None):
+    def getReader(self, key=None, replace=False):
 
-        return InputStreamReader(self.getInputStream(key), self.encoding)
+        return InputStreamReader(self.getInputStream(key),
+                                 self.encoding, replace)
 
-    def getPlainTextReader(self, key=None):
+    def getPlainTextReader(self, key=None, replace=False):
 
         if self.mimetype in Lob._readers:
-            return Lob._readers[self.mimetype](self, key)
+            return Lob._readers[self.mimetype](self, key, replace)
 
         return NotImplementedError, "Converting mimetype '%s' to plain text" %(self.mimetype)
 
     _readers = {
-        'text/html': lambda self, key: HTMLReader(self.getInputStream(key),
-                                                  self.encoding), 
-        'text/xhtml': lambda self, key: HTMLReader(self.getInputStream(key),
-                                                   self.encoding),
-        'text/plain': lambda self, key: self.getReader(key),
+        'text/html': lambda self, key, replace: HTMLReader(self.getInputStream(key), self.encoding, replace), 
+        'text/xhtml': lambda self, key, replace: HTMLReader(self.getInputStream(key), self.encoding, replace),
+        'text/plain': lambda self, key, replace: self.getReader(key, replace),
 
-        'text/vnd.osaf-stream64': lambda self, key: InputStreamReader(Base64InputStream(self.getInputStream(key)), self.encoding)
+        'text/vnd.osaf-stream64': lambda self, key, replace: InputStreamReader(Base64InputStream(self.getInputStream(key)), self.encoding, replace)
     }

Index: chandler/repository/util/Streams.py
diff -u chandler/repository/util/Streams.py:1.19 chandler/repository/util/Streams.py:1.20
--- chandler/repository/util/Streams.py:1.19	Tue Jan 18 13:29:57 2005
+++ chandler/repository/util/Streams.py	Tue Feb 15 16:29:21 2005
@@ -1,6 +1,6 @@
 
-__revision__  = "$Revision: 1.19 $"
-__date__      = "$Date: 2005/01/18 21:29:57 $"
+__revision__  = "$Revision: 1.20 $"
+__date__      = "$Date: 2005/02/16 00:29:21 $"
 __copyright__ = "Copyright (c) 2003-2004 Open Source Applications Foundation"
 __license__   = "http://osafoundation.org/Chandler_0.1_license_terms.htm"
 
@@ -541,16 +541,18 @@
 
 class OutputStreamWriter(object):
 
-    def __init__(self, outputStream, encoding):
+    def __init__(self, outputStream, encoding, replace=False):
 
         super(OutputStreamWriter, self).__init__()
+
         self.outputStream = outputStream
         self.encoding = encoding or 'utf-8'
+        self.errorHandler = replace and 'replace' or 'strict'
 
     def write(self, text):
 
         if isinstance(text, unicode):
-            text = text.encode(self.encoding)
+            text = text.encode(self.encoding, self.errorHandler)
 
         self.outputStream.write(text)
 
@@ -565,11 +567,13 @@
 
 class InputStreamReader(object):
 
-    def __init__(self, inputStream, encoding):
+    def __init__(self, inputStream, encoding, replace=False):
 
         super(InputStreamReader, self).__init__()
+
         self.inputStream = inputStream
         self.encoding = encoding or 'utf-8'
+        self.errorHandler = replace and 'replace' or 'strict'
 
     def _read(self, length):
 
@@ -578,7 +582,7 @@
     def read(self, length=-1):
 
         text = self._read(length)
-        text = unicode(text, self.encoding)
+        text = unicode(text, self.encoding, self.errorHandler)
 
         return text
 
@@ -589,10 +593,15 @@
 
 class StringReader(object):
 
-    def __init__(self, unicodeText):
+    def __init__(self, text, encoding=None, replace=False):
 
         super(StringReader, self).__init__()
-        self.unicodeText = unicode(unicodeText)
+
+        if not isinstance(text, unicode):
+            self.unicodeText = unicode(text, encoding or 'utf-8',
+                                       replace and 'replace' or 'strict')
+        else:
+            self.unicodeText = text
 
     def read(self, length=-1):
 
@@ -615,9 +624,9 @@
 
 class HTMLReader(InputStreamReader):
 
-    def __init__(self, inputStream, encoding):
+    def __init__(self, inputStream, encoding, replace=False):
 
-        super(HTMLReader, self).__init__(inputStream, encoding)
+        super(HTMLReader, self).__init__(inputStream, encoding, replace)
 
         class htmlParser(HTMLParser):
 



More information about the Commits mailing list