Package translate :: Package storage :: Module base

[hide private]
[frames] | no frames]

# Source Code for Module translate.storage.base

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  #
      4  # Copyright 2006-2009 Zuza Software Foundation
      5  #
      6  # This file is part of the Translate Toolkit.
      7  #
      8  # This program is free software; you can redistribute it and/or modify
      9  # it under the terms of the GNU General Public License as published by
     10  # the Free Software Foundation; either version 2 of the License, or
     11  # (at your option) any later version.
     12  #
     13  # This program is distributed in the hope that it will be useful,
     14  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     16  # GNU General Public License for more details.
     17  #
     18  # You should have received a copy of the GNU General Public License
     19  # along with this program; if not, see <http://www.gnu.org/licenses/>.
     20
     21  """Base classes for storage interfaces.
     22
     23  @organization: Zuza Software Foundation
     24  @copyright: 2006-2009 Zuza Software Foundation
     25  @license: U{GPL <http://www.fsf.org/licensing/licenses/gpl.html>}
     26  """
     27
     28  try:
     29      import cPickle as pickle
     30  except:
     31      import pickle
     32  from exceptions import NotImplementedError
     33  import translate.i18n
     34  from translate.storage.placeables import StringElem, general, parse as rich_parse
     35  from translate.misc.typecheck import accepts, Self, IsOneOf
     36  from translate.misc.multistring import multistring
     37
     38 -def force_override(method, baseclass):
     39      """Forces derived classes to override method."""
     40
     41      if type(method.im_self) == type(baseclass):
     42          # then this is a classmethod and im_self is the actual class
     43          actualclass = method.im_self
     44      else:
     45          actualclass = method.im_class
     46      if actualclass != baseclass:
     47          raise NotImplementedError(
     48              "%s does not reimplement %s as required by %s" % \
     49              (actualclass.__name__, method.__name__, baseclass.__name__)
     50          )
     51
     52
     53 -class ParseError(Exception):
     54 -    def __init__(self, inner_exc):
     55          self.inner_exc = inner_exc
     56
     57 -    def __str__(self):
     58          return repr(self.inner_exc)
     59
     60
     61 -class TranslationUnit(object):
     62      """Base class for translation units.
     63
     64      Our concept of a I{translation unit} is influenced heavily by XLIFF:
     65      U{http://www.oasis-open.org/committees/xliff/documents/xliff-specification.htm}
     66
     67      As such most of the method- and variable names borrows from XLIFF terminology.
     68
     69      A translation unit consists of the following:
     70        - A I{source} string. This is the original translatable text.
     71        - A I{target} string. This is the translation of the I{source}.
     72        - Zero or more I{notes} on the unit. Notes would typically be some
     73          comments from a translator on the unit, or some comments originating from
     74          the source code.
     75        - Zero or more I{locations}. Locations indicate where in the original
     76          source code this unit came from.
     77        - Zero or more I{errors}. Some tools (eg. L{pofilter <filters.pofilter>}) can run checks on
     78          translations and produce error messages.
     79
     80      @group Source: *source*
     81      @group Target: *target*
     82      @group Notes: *note*
     83      @group Locations: *location*
     84      @group Errors: *error*
     85      """
     86
     87      rich_parsers = []
     88      """A list of functions to use for parsing a string into a rich string tree."""
     89
     90 -    def __init__(self, source):
     91          """Constructs a TranslationUnit containing the given source string."""
     92          self.notes = ""
     93          self._store = None
     94          self.source = source
     95          self._target = None
     96          self._rich_source = None
     97          self._rich_target = None
     98
     99 -    def __eq__(self, other):
    100          """Compares two TranslationUnits.
    101
    102          @type other: L{TranslationUnit}
    103          @param other: Another L{TranslationUnit}
    104          @rtype: Boolean
    105          @return: Returns True if the supplied TranslationUnit equals this unit.
    106          """
    107          return self.source == other.source and self.target == other.target
    108
    109 -    def __str__(self):
    110          """Converts to a string representation that can be parsed back using L{parsestring()}."""
    111          # no point in pickling store object, so let's hide it for a while.
    112          store = getattr(self, "_store", None)
    113          self._store = None
    114          dump = pickle.dumps(self)
    115          self._store = store
    116          return dump
    117
    118 -    def rich_to_multistring(cls, elem_list):
    119          """Convert a "rich" string tree to a C{multistring}:
    120
    121             >>> from translate.storage.placeables.interfaces import X
    122             >>> rich = [StringElem(['foo', X(id='xxx', sub=[' ']), 'bar'])]
    123             >>> TranslationUnit.rich_to_multistring(rich)
    124             multistring(u'foo bar')
    125          """
    126          return multistring([unicode(elem) for elem in elem_list])
    127      rich_to_multistring = classmethod(rich_to_multistring)
    128
    129 -    def multistring_to_rich(cls, mulstring):
    130          """Convert a multistring to a list of "rich" string trees:
    131
    132             >>> target = multistring([u'foo', u'bar', u'baz'])
    133             >>> TranslationUnit.multistring_to_rich(target)
    134             [<StringElem([<StringElem([u'foo'])>])>,
    135              <StringElem([<StringElem([u'bar'])>])>,
    136              <StringElem([<StringElem([u'baz'])>])>]
    137          """
    138          if isinstance(mulstring, multistring):
    139              return [rich_parse(s, cls.rich_parsers) for s in mulstring.strings]
    140          return [rich_parse(mulstring, cls.rich_parsers)]
    141
    142 -    def setsource(self, source):
    143          """Sets the source string to the given value."""
    144          self._rich_source = None
    145          self._source = source
    146      source = property(lambda self: self._source, setsource)
    147
    148 -    def settarget(self, target):
    149          """Sets the target string to the given value."""
    150          self._rich_target = None
    151          self._target = target
    152      target = property(lambda self: self._target, settarget)
    153
    154 -    def _get_rich_source(self):
    155          if self._rich_source is None:
    156              self._rich_source = self.multistring_to_rich(self.source)
    157          return self._rich_source
    158 -    def _set_rich_source(self, value):
    159          if not hasattr(value, '__iter__'):
    160              raise ValueError('value must be iterable')
    161          if len(value) < 1:
    162              raise ValueError('value must have at least one element.')
    163          if not isinstance(value[0], StringElem):
    164              raise ValueError('value[0] must be of type StringElem.')
    165          self._rich_source = list(value)
    166          self.source = self.rich_to_multistring(value)
    167      rich_source = property(_get_rich_source, _set_rich_source)
    168      """ @see: rich_to_multistring
    169          @see: multistring_to_rich"""
    170
    171 -    def _get_rich_target(self):
    172          if self._rich_target is None:
    173              self._rich_target = self.multistring_to_rich(self.target)
    174          return self._rich_target
    175 -    def _set_rich_target(self, value):
    176          if not hasattr(value, '__iter__'):
    177              raise ValueError('value must be iterable')
    178          if len(value) < 1:
    179              raise ValueError('value must have at least one element.')
    180          if not isinstance(value[0], StringElem):
    181              raise ValueError('value[0] must be of type StringElem.')
    182          self._rich_target = list(value)
    183          self.target = self.rich_to_multistring(value)
    184      rich_target = property(_get_rich_target, _set_rich_target)
    185      """ @see: rich_to_multistring
    186          @see: multistring_to_rich"""
    187
    188 -    def gettargetlen(self):
    189          """Returns the length of the target string.
    190
    191          @note: Plural forms might be combined.
    192          @rtype: Integer
    193          """
    194          length = len(self.target or "")
    195          strings = getattr(self.target, "strings", [])
    196          if strings:
    197              length += sum([len(pluralform) for pluralform in strings[1:]])
    198          return length
    199
    200 -    def getid(self):
    201          """A unique identifier for this unit.
    202
    203          @rtype: string
    204          @return: an identifier for this unit that is unique in the store
    205
    206          Derived classes should override this in a way that guarantees a unique
    207          identifier for each unit in the store.
    208          """
    209          return self.source
    210
    211 -    def setid(self, value):
    212          """Sets the unique identified for this unit.
    213
    214          only implemented if format allows ids independant from other
    215          unit properties like source or context"""
    216          pass
    217
    218 -    def getlocations(self):
    219          """A list of source code locations.
    220
    221          @note: Shouldn't be implemented if the format doesn't support it.
    222          @rtype: List
    223          """
    224          return []
    225
    226 -    def addlocation(self, location):
    227          """Add one location to the list of locations.
    228
    229          @note: Shouldn't be implemented if the format doesn't support it.
    230          """
    231          pass
    232
    233 -    def addlocations(self, location):
    234          """Add a location or a list of locations.
    235
    236          @note: Most classes shouldn't need to implement this,
    237                 but should rather implement L{addlocation()}.
    238          @warning: This method might be removed in future.
    239          """
    240          if isinstance(location, list):
    241              for item in location:
    242                  self.addlocation(item)
    243          else:
    244              self.addlocation(location)
    245
    246 -    def getcontext(self):
    247          """Get the message context."""
    248          return ""
    249
    250 -    def setcontext(self, context):
    251          """Set the message context"""
    252          pass
    253
    254 -    def getnotes(self, origin=None):
    255          """Returns all notes about this unit.
    256
    257          It will probably be freeform text or something reasonable that can be
    258          synthesised by the format.
    259          It should not include location comments (see L{getlocations()}).
    260          """
    261          return getattr(self, "notes", "")
    262
    263 -    def addnote(self, text, origin=None, position="append"):
    264          """Adds a note (comment).
    265
    266          @type text: string
    267          @param text: Usually just a sentence or two.
    268          @type origin: string
    269          @param origin: Specifies who/where the comment comes from.
    270                         Origin can be one of the following text strings:
    271                           - 'translator'
    272                           - 'developer', 'programmer', 'source code' (synonyms)
    273          """
    274          if getattr(self, "notes", None):
    275              self.notes += '\n'+text
    276          else:
    277              self.notes = text
    278
    279 -    def removenotes(self):
    280          """Remove all the translator's notes."""
    281          self.notes = u''
    282
    283 -    def adderror(self, errorname, errortext):
    284          """Adds an error message to this unit.
    285
    286          @type errorname: string
    287          @param errorname: A single word to id the error.
    288          @type errortext: string
    289          @param errortext: The text describing the error.
    290          """
    291          pass
    292
    293 -    def geterrors(self):
    294          """Get all error messages.
    295
    296          @rtype: Dictionary
    297          """
    298          return {}
    299
    300 -    def markreviewneeded(self, needsreview=True, explanation=None):
    301          """Marks the unit to indicate whether it needs review.
    302
    303          @keyword needsreview: Defaults to True.
    304          @keyword explanation: Adds an optional explanation as a note.
    305          """
    306          pass
    307
    308 -    def istranslated(self):
    309          """Indicates whether this unit is translated.
    310
    311          This should be used rather than deducing it from .target,
    312          to ensure that other classes can implement more functionality
    313          (as XLIFF does).
    314          """
    315          return bool(self.target) and not self.isfuzzy()
    316
    317 -    def istranslatable(self):
    318          """Indicates whether this unit can be translated.
    319
    320          This should be used to distinguish real units for translation from
    321          header, obsolete, binary or other blank units.
    322          """
    323          return True
    324
    325 -    def isfuzzy(self):
    326          """Indicates whether this unit is fuzzy."""
    327          return False
    328
    329 -    def markfuzzy(self, value=True):
    330          """Marks the unit as fuzzy or not."""
    331          pass
    332
    333 -    def isobsolete(self):
    334          """indicate whether a unit is obsolete"""
    335          return False
    336
    337 -    def makeobsolete(self):
    338          """Make a unit obsolete"""
    339          pass
    340
    341 -    def isheader(self):
    342          """Indicates whether this unit is a header."""
    343          return False
    344
    345 -    def isreview(self):
    346          """Indicates whether this unit needs review."""
    347          return False
    348
    349 -    def isblank(self):
    350          """Used to see if this unit has no source or target string.
    351
    352          @note: This is probably used more to find translatable units,
    353          and we might want to move in that direction rather and get rid of this.
    354          """
    355          return not (self.source or self.target)
    356
    357 -    def hasplural(self):
    358          """Tells whether or not this specific unit has plural strings."""
    359          #TODO: Reconsider
    360          return False
    361
    362 -    def getsourcelanguage(self):
    363          return getattr(self._store, "sourcelanguage", "en")
    364
    365 -    def gettargetlanguage(self):
    366          return getattr(self._store, "targetlanguage", None)
    367
    368 -    def merge(self, otherunit, overwrite=False, comments=True, authoritative=False):
    369          """Do basic format agnostic merging."""
    370          if not self.target or overwrite:
    371              self.rich_target = otherunit.rich_target
    372
    373 -    def unit_iter(self):
    374          """Iterator that only returns this unit."""
    375          yield self
    376
    377 -    def getunits(self):
    378          """This unit in a list."""
    379          return [self]
    380
    381 -    def buildfromunit(cls, unit):
    382          """Build a native unit from a foreign unit, preserving as much
    383          information as possible."""
    384          if type(unit) == cls and hasattr(unit, "copy") and callable(unit.copy):
    385              return unit.copy()
    386          newunit = cls(unit.source)
    387          newunit.target = unit.target
    388          newunit.markfuzzy(unit.isfuzzy())
    389          locations = unit.getlocations()
    390          if locations:
    391              newunit.addlocations(locations)
    392          notes = unit.getnotes()
    393          if notes:
    394              newunit.addnote(notes)
    395          return newunit
    396      buildfromunit = classmethod(buildfromunit)
    397
    398      xid = property(lambda self: None, lambda self, value: None)
    399      rid = property(lambda self: None, lambda self, value: None)
    400
    401
    402 -class TranslationStore(object):
    403      """Base class for stores for multiple translation units of type UnitClass."""
    404
    405      UnitClass = TranslationUnit
    406      """The class of units that will be instantiated and used by this class"""
    407      Name = "Base translation store"
    408      """The human usable name of this store type"""
    409      Mimetypes = None
    410      """A list of MIME types associated with this store type"""
    411      Extensions = None
    412      """A list of file extentions associated with this store type"""
    413      _binary = False
    414      """Indicates whether a file should be accessed as a binary file."""
    415      suggestions_in_format = False
    416      """Indicates if format can store suggestions and alternative translation for a unit"""
    417
    418 -    def __init__(self, unitclass=None):
    419          """Constructs a blank TranslationStore."""
    420          self.units = []
    421          self.sourcelanguage = None
    422          self.targetlanguage = None
    423          if unitclass:
    424              self.UnitClass = unitclass
    425          super(TranslationStore, self).__init__()
    426
    427 -    def getsourcelanguage(self):
    428          """Gets the source language for this store"""
    429          return self.sourcelanguage
    430
    431 -    def setsourcelanguage(self, sourcelanguage):
    432          """Sets the source language for this store"""
    433          self.sourcelanguage = sourcelanguage
    434
    435 -    def gettargetlanguage(self):
    436          """Gets the target language for this store"""
    437          return self.targetlanguage
    438
    439 -    def settargetlanguage(self, targetlanguage):
    440          """Sets the target language for this store"""
    441          self.targetlanguage = targetlanguage
    442
    443 -    def unit_iter(self):
    444          """Iterator over all the units in this store."""
    445          for unit in self.units:
    446              yield unit
    447
    448 -    def getunits(self):
    449          """Return a list of all units in this store."""
    450          return [unit for unit in self.unit_iter()]
    451
    452 -    def addunit(self, unit):
    453          """Appends the given unit to the object's list of units.
    454
    455          This method should always be used rather than trying to modify the
    456          list manually.
    457
    458          @type unit: L{TranslationUnit}
    459          @param unit: The unit that will be added.
    460          """
    461          unit._store = self
    462          self.units.append(unit)
    463
    464 -    def addsourceunit(self, source):
    465          """Adds and returns a new unit with the given source string.
    466
    467          @rtype: L{TranslationUnit}
    468          """
    469          unit = self.UnitClass(source)
    470          self.addunit(unit)
    471          return unit
    472
    473 -    def findid(self, id):
    474          """find unit with matching id by checking id_index"""
    475          self.require_index()
    476          return self.id_index.get(id, None)
    477
    478 -    def findunit(self, source):
    479          """Finds the unit with the given source string.
    480
    481          @rtype: L{TranslationUnit} or None
    482          """
    483          if len(getattr(self, "sourceindex", [])):
    484              if source in self.sourceindex:
    485                  return self.sourceindex[source][0]
    486          else:
    487              for unit in self.units:
    488                  if unit.source == source:
    489                      return unit
    490          return None
    491
    492
    493 -    def findunits(self, source):
    494          """Finds the units with the given source string.
    495
    496          @rtype: L{TranslationUnit} or None
    497          """
    498          if len(getattr(self, "sourceindex", [])):
    499              if source in self.sourceindex:
    500                  return self.sourceindex[source]
    501          else:
    502              #FIXME: maybe we should generate index here instead since
    503              #we'll scan all units anyway
    504              result = []
    505              for unit in self.units:
    506                  if unit.source == source:
    507                      result.append(unit)
    508              return result
    509          return None
    510
    511 -    def translate(self, source):
    512          """Returns the translated string for a given source string.
    513
    514          @rtype: String or None
    515          """
    516          unit = self.findunit(source)
    517          if unit and unit.target:
    518              return unit.target
    519          else:
    520              return None
    521
    522 -    def remove_unit_from_index(self, unit):
    523          """Remove a unit from source and locaton indexes"""
    524          def remove_unit(source):
    525              if source in self.sourceindex:
    526                  try:
    527                      self.sourceindex[source].remove(unit)
    528                      if len(self.sourceindex[source]) == 0:
    529                          del(self.sourceindex[source])
    530                  except ValueError:
    531                      pass
    532
    533          if unit.hasplural():
    534              for source in unit.source.strings:
    535                  remove_unit(source)
    536          else:
    537              remove_unit(unit.source)
    538
    539          for location in unit.getlocations():
    540              if location in self.locationindex and self.locationindex[location] is not None \
    541                     and self.locationindex[location] == unit:
    542                  del(self.locationindex[location])
    543
    544
    545 -    def add_unit_to_index(self, unit):
    546          """Add a unit to source and location idexes"""
    547          self.id_index[unit.getid()] = unit
    548
    549          def insert_unit(source):
    550              if not source in self.sourceindex:
    551                  self.sourceindex[source] = [unit]
    552              else:
    553                  self.sourceindex[source].append(unit)
    554
    555          if unit.hasplural():
    556              for source in unit.source.strings:
    557                  insert_unit(source)
    558          else:
    559              insert_unit(unit.source)
    560
    561          for location in unit.getlocations():
    562              if location in self.locationindex:
    563                  # if sources aren't unique, don't use them
    564                  #FIXME: maybe better store a list of units like sourceindex
    565                  self.locationindex[location] = None
    566              else:
    567                  self.locationindex[location] = unit
    568
    569 -    def makeindex(self):
    570          """Indexes the items in this store. At least .sourceindex should be usefull."""
    571          self.locationindex = {}
    572          self.sourceindex = {}
    573          self.id_index = {}
    574          for index, unit in enumerate(self.units):
    575              unit.index = index
    576              if unit.istranslatable():
    577                  self.add_unit_to_index(unit)
    578
    579 -    def require_index(self):
    580          """make sure source index exists"""
    581          if not hasattr(self, "sourceindex"):
    582              self.makeindex()
    583
    584 -    def getids(self):
    585          """return a list of unit ids"""
    586          self.require_index()
    587          return self.id_index.keys()
    588
    589 -    def __getstate__(self):
    590          odict = self.__dict__.copy()
    591          odict['fileobj'] = None
    592          return odict
    593
    594 -    def __setstate__(self, dict):
    595          self.__dict__.update(dict)
    596          if getattr(self, "filename", False):
    597              self.fileobj = open(self.filename)
    598
    599 -    def __str__(self):
    600          """Converts to a string representation that can be parsed back using L{parsestring()}."""
    601          # We can't pickle fileobj if it is there, so let's hide it for a while.
    602          fileobj = getattr(self, "fileobj", None)
    603          self.fileobj = None
    604          dump = pickle.dumps(self)
    605          self.fileobj = fileobj
    606          return dump
    607
    608 -    def isempty(self):
    609          """Returns True if the object doesn't contain any translation units."""
    610          if len(self.units) == 0:
    611              return True
    612          for unit in self.units:
    613              if unit.istranslatable():
    614                  return False
    615          return True
    616
    617 -    def _assignname(self):
    618          """Tries to work out what the name of the filesystem file is and
    619          assigns it to .filename."""
    620          fileobj = getattr(self, "fileobj", None)
    621          if fileobj:
    622              filename = getattr(fileobj, "name", getattr(fileobj, "filename", None))
    623              if filename:
    624                  self.filename = filename
    625
    626 -    def parsestring(cls, storestring):
    627          """Converts the string representation back to an object."""
    628          newstore = cls()
    629          if storestring:
    630              newstore.parse(storestring)
    631          return newstore
    632      parsestring = classmethod(parsestring)
    633
    634 -    def parse(self, data):
    635          """parser to process the given source string"""
    636          self.units = pickle.loads(data).units
    637
    638 -    def savefile(self, storefile):
    639          """Writes the string representation to the given file (or filename)."""
    640          if isinstance(storefile, basestring):
    641              mode = 'w'
    642              if self._binary:
    643                  mode = 'wb'
    644              storefile = open(storefile, mode)
    645          self.fileobj = storefile
    646          self._assignname()
    647          storestring = str(self)
    648          storefile.write(storestring)
    649          storefile.close()
    650
    651 -    def save(self):
    652          """Save to the file that data was originally read from, if available."""
    653          fileobj = getattr(self, "fileobj", None)
    654          mode = 'w'
    655          if self._binary:
    656              mode = 'wb'
    657          if not fileobj:
    658              filename = getattr(self, "filename", None)
    659              if filename:
    660                  fileobj = file(filename, mode)
    661          else:
    662              fileobj.close()
    663              filename = getattr(fileobj, "name", getattr(fileobj, "filename", None))
    664              if not filename:
    665                  raise ValueError("No file or filename to save to")
    666              fileobj = fileobj.__class__(filename, mode)
    667          self.savefile(fileobj)
    668
    669 -    def parsefile(cls, storefile):
    670          """Reads the given file (or opens the given filename) and parses back to an object."""
    671          mode = 'r'
    672          if cls._binary:
    673              mode = 'rb'
    674          if isinstance(storefile, basestring):
    675              storefile = open(storefile, mode)
    676          mode = getattr(storefile, "mode", mode)
    677          #For some reason GzipFile returns 1, so we have to test for that here
    678          if mode == 1 or "r" in mode:
    679              storestring = storefile.read()
    680              storefile.close()
    681          else:
    682              storestring = ""
    683          newstore = cls.parsestring(storestring)
    684          newstore.fileobj = storefile
    685          newstore._assignname()
    686          return newstore
    687      parsefile = classmethod(parsefile)
    688