1   
  2   
  3   
  4   
  5   
  6   
  7   
  8   
  9   
 10   
 11   
 12   
 13   
 14   
 15   
 16   
 17   
 18   
 19   
 20   
 21   
 22  """Classes that hold units of .po files (pounit) or entire files (pofile). 
 23   
 24  Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and 
 25  many other projects. 
 26   
 27  This uses libgettextpo from the gettext package. Any version before 0.17 will 
 28  at least cause some subtle bugs or may not work at all. Developers might want 
 29  to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext 
 30  package for the public API of the library. 
 31  """ 
 32   
 33  from translate.misc.multistring import multistring 
 34  from translate.storage import pocommon 
 35  from translate.misc import quote 
 36  from translate.lang import data 
 37  from ctypes import * 
 38  import ctypes.util 
 39  try: 
 40      import cStringIO as StringIO 
 41  except ImportError: 
 42      import StringIO 
 43  import os 
 44  import pypo 
 45  import re 
 46  import sys 
 47   
 48  lsep = " " 
 49  """Seperator for #: entries""" 
 50   
 51  STRING = c_char_p 
 52   
 53   
 56   
 57   
 58  xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING) 
 59  xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING) 
 60   
 61   
 62   
 64      _fields_ = [('xerror', xerror_prototype), 
 65                  ('xerror2', xerror2_prototype)] 
  66   
 68      _fields_ = [ 
 69      ('error', CFUNCTYPE(None, c_int, c_int, STRING)), 
 70      ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)), 
 71      ('multiline_warning', CFUNCTYPE(None, STRING, STRING)), 
 72      ('multiline_error', CFUNCTYPE(None, STRING, STRING)), 
 73  ] 
  74   
 75   
 76 -def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_text): 
  77      print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, column, multilint_p, message_text 
 78      if severity >= 1: 
 79          raise ValueError(message_text) 
  80   
 81 -def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2): 
  82      print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2 
 83      if severity >= 1: 
 84          raise ValueError(message_text1) 
  85   
 86   
 87   
 88   
 89  gpo = None 
 90   
 91   
 92  names = ['gettextpo', 'libgettextpo'] 
 93  for name in names: 
 94      lib_location = ctypes.util.find_library(name) 
 95      if lib_location: 
 96          gpo = cdll.LoadLibrary(lib_location) 
 97          if gpo: 
 98              break 
 99  else: 
100       
101       
102      try: 
103          gpo = cdll.LoadLibrary('libgettextpo.so') 
104      except OSError, e: 
105          raise ImportError("gettext PO library not found") 
106   
107   
108   
109  gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)] 
110  gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)] 
111  gpo.po_file_write_v2.retype = c_int 
112   
113   
114  gpo.po_file_domain_header.restype = STRING 
115  gpo.po_header_field.restype = STRING 
116  gpo.po_header_field.argtypes = [STRING, STRING] 
117   
118   
119  gpo.po_filepos_file.restype = STRING 
120  gpo.po_message_filepos.restype = c_int 
121  gpo.po_message_filepos.argtypes = [c_int, c_int] 
122  gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int] 
123   
124   
125  gpo.po_message_comments.restype = STRING 
126  gpo.po_message_extracted_comments.restype = STRING 
127  gpo.po_message_prev_msgctxt.restype = STRING 
128  gpo.po_message_prev_msgid.restype = STRING 
129  gpo.po_message_prev_msgid_plural.restype = STRING 
130  gpo.po_message_is_format.restype = c_int 
131  gpo.po_message_msgctxt.restype = STRING 
132  gpo.po_message_msgid.restype = STRING 
133  gpo.po_message_msgid_plural.restype = STRING 
134  gpo.po_message_msgstr.restype = STRING 
135  gpo.po_message_msgstr_plural.restype = STRING 
136   
137   
138  gpo.po_message_set_comments.argtypes = [c_int, STRING] 
139  gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING] 
140  gpo.po_message_set_fuzzy.argtypes = [c_int, c_int] 
141  gpo.po_message_set_msgctxt.argtypes = [c_int, STRING] 
142   
143   
144  xerror_handler = po_xerror_handler() 
145  xerror_handler.xerror = xerror_prototype(xerror_cb) 
146  xerror_handler.xerror2 = xerror2_prototype(xerror2_cb) 
147   
150   
153   
156   
159   
160 -class pounit(pocommon.pounit): 
 181      msgid_plural = property(None, setmsgid_plural) 
182   
184          def remove_msgid_comments(text): 
185              if not text: 
186                  return text 
187              if text.startswith("_:"): 
188                  remainder = re.search(r"_: .*\n(.*)", text) 
189                  if remainder: 
190                      return remainder.group(1) 
191                  else: 
192                      return u"" 
193              else: 
194                  return text 
 195          singular = remove_msgid_comments(gpo.po_message_msgid(self._gpo_message)) 
196          if singular: 
197              multi = multistring(singular, self._encoding) 
198              if self.hasplural(): 
199                  pluralform = gpo.po_message_msgid_plural(self._gpo_message) 
200                  if isinstance(pluralform, str): 
201                      pluralform = pluralform.decode(self._encoding) 
202                  multi.strings.append(pluralform) 
203              return multi 
204          else: 
205              return u"" 
206   
219               
220      source = property(getsource, setsource) 
221   
223          if self.hasplural(): 
224              plurals = [] 
225              nplural = 0 
226              plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural) 
227              while plural: 
228                  plurals.append(plural) 
229                  nplural += 1 
230                  plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural) 
231              if plurals: 
232                  multi = multistring(plurals, encoding=self._encoding) 
233              else: 
234                  multi = multistring(u"") 
235          else: 
236              multi = multistring(gpo.po_message_msgstr(self._gpo_message) or u"", encoding=self._encoding) 
237          return multi 
 238   
240          if self.hasplural(): 
241              if isinstance(target, multistring): 
242                  target = target.strings 
243              elif isinstance(target, basestring): 
244                  target = [target] 
245          elif isinstance(target,(dict, list)): 
246              if len(target) == 1: 
247                  target = target[0] 
248              else: 
249                  raise ValueError("po msgid element has no plural but msgstr has %d elements (%s)" % (len(target), target)) 
250          if isinstance(target, (dict, list)): 
251              i = 0 
252              message = gpo.po_message_msgstr_plural(self._gpo_message, i) 
253              while message is not None: 
254                  gpo.po_message_set_msgstr_plural(self._gpo_message, i, None) 
255                  i += 1 
256                  message = gpo.po_message_msgstr_plural(self._gpo_message, i) 
257          if isinstance(target, list): 
258              for i in range(len(target)): 
259                  targetstring = target[i] 
260                  if isinstance(targetstring, unicode): 
261                      targetstring = targetstring.encode(self._encoding) 
262                  gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring) 
263          elif isinstance(target, dict): 
264              for i, targetstring in enumerate(target.itervalues()): 
265                  gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring) 
266          else: 
267              if isinstance(target, unicode): 
268                  target = target.encode(self._encoding) 
269              if target is None: 
270                  gpo.po_message_set_msgstr(self._gpo_message, "") 
271              else: 
272                  gpo.po_message_set_msgstr(self._gpo_message, target) 
 273      target = property(gettarget, settarget) 
274   
276          """The unique identifier for this unit according to the convensions in 
277          .mo files.""" 
278          id = gpo.po_message_msgid(self._gpo_message) 
279           
280           
281           
282           
283   
284   
285   
286          context = gpo.po_message_msgctxt(self._gpo_message) 
287          if context: 
288              id = "%s\04%s" % (context, id) 
289          return id or "" 
 290   
292          if origin == None: 
293              comments = gpo.po_message_comments(self._gpo_message) + \ 
294                         gpo.po_message_extracted_comments(self._gpo_message) 
295          elif origin == "translator": 
296              comments = gpo.po_message_comments(self._gpo_message) 
297          elif origin in ["programmer", "developer", "source code"]: 
298              comments = gpo.po_message_extracted_comments(self._gpo_message) 
299          else: 
300              raise ValueError("Comment type not valid") 
301           
302          if comments: 
303              comments = "\n".join([line[1:] for line in comments.split("\n")]) 
304           
305          return comments[:-1].decode(self._encoding) 
 306   
307 -    def addnote(self, text, origin=None, position="append"): 
 308          if not text: 
309              return 
310          text = data.forceunicode(text) 
311          oldnotes = self.getnotes(origin) 
312          newnotes = None 
313          if oldnotes: 
314              if position == "append": 
315                  newnotes = oldnotes + "\n" + text 
316              elif position == "merge": 
317                  if oldnotes != text: 
318                      oldnoteslist = oldnotes.split("\n") 
319                      for newline in text.split("\n"): 
320                          newline = newline.rstrip() 
321                           
322                          if newline not in oldnotes or len(newline) < 5: 
323                              oldnoteslist.append(newline) 
324                      newnotes = "\n".join(oldnoteslist) 
325              else: 
326                  newnotes = text + '\n' + oldnotes 
327          else: 
328              newnotes = "\n".join([line.rstrip() for line in text.split("\n")]) 
329           
330          if newnotes: 
331              newlines = [] 
332              for line in newnotes.split("\n"): 
333                  if line: 
334                      newlines.append(" " + line) 
335                  else: 
336                      newlines.append(line) 
337              newnotes = "\n".join(newlines) 
338              if origin in ["programmer", "developer", "source code"]: 
339                  gpo.po_message_set_extracted_comments(self._gpo_message, newnotes) 
340              else: 
341                  gpo.po_message_set_comments(self._gpo_message, newnotes) 
 342   
344          gpo.po_message_set_comments(self._gpo_message, "") 
 345   
347          newpo = self.__class__() 
348          newpo._gpo_message = self._gpo_message 
349          return newpo 
 350   
351 -    def merge(self, otherpo, overwrite=False, comments=True, authoritative=False): 
 386   
388           
389           
390          return self.getid() == "" and len(self.target) > 0 
 391   
394   
397   
404   
407   
410   
412          return gpo.po_message_is_fuzzy(self._gpo_message) 
 413   
415          gpo.po_message_set_fuzzy(self._gpo_message, present) 
 416   
419   
421          return gpo.po_message_is_obsolete(self._gpo_message) 
 422   
424           
425           
426          gpo.po_message_set_obsolete(self._gpo_message, True) 
 427   
429          gpo.po_message_set_obsolete(self._gpo_message, False) 
 430   
432          return gpo.po_message_msgid_plural(self._gpo_message) is not None 
 433   
449   
454   
456          locations = [] 
457          i = 0 
458          location = gpo.po_message_filepos(self._gpo_message, i) 
459          while location: 
460              locname = gpo.po_filepos_file(location) 
461              locline = gpo.po_filepos_start_line(location) 
462              if locline == -1: 
463                  locstring = locname 
464              else: 
465                  locstring = locname + ":" + str(locline) 
466              locations.append(locstring) 
467              i += 1 
468              location = gpo.po_message_filepos(self._gpo_message, i) 
469          return locations 
 470   
472          for loc in location.split(): 
473              parts = loc.split(":") 
474              file = parts[0] 
475              if len(parts) == 2: 
476                  line = int(parts[1]) 
477              else: 
478                  line = -1 
479              gpo.po_message_add_filepos(self._gpo_message, file, line) 
 480   
481 -    def getcontext(self): 
 482          msgctxt = gpo.po_message_msgctxt(self._gpo_message) 
483          msgidcomment = self._extract_msgidcomments() 
484          if msgctxt: 
485              return msgctxt + msgidcomment 
486          else: 
487              return msgidcomment 
 488   
489 -class pofile(pocommon.pofile): 
 490      UnitClass = pounit 
492          self.UnitClass = unitclass 
493          pocommon.pofile.__init__(self, unitclass=unitclass) 
494          self._gpo_memory_file = None 
495          self._gpo_message_iterator = None 
496          self._encoding = encodingToUse(encoding) 
497          if inputfile is not None: 
498              self.parse(inputfile) 
499          else: 
500              self._gpo_memory_file = gpo.po_file_create() 
501              self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None) 
 502   
504          gpo.po_message_insert(self._gpo_message_iterator, unit._gpo_message) 
505          self.units.append(unit) 
 506   
508          """make sure each msgid is unique ; merge comments etc from duplicates into original""" 
509          msgiddict = {} 
510          uniqueunits = [] 
511           
512           
513          markedpos = [] 
514          def addcomment(thepo): 
515              thepo.msgidcomment = " ".join(thepo.getlocations()) 
516              markedpos.append(thepo) 
 517          for thepo in self.units: 
518              if thepo.isheader(): 
519                  uniqueunits.append(thepo) 
520                  continue 
521              if duplicatestyle.startswith("msgid_comment"): 
522                  msgid = thepo._extract_msgidcomments() + thepo.source 
523              else: 
524                  msgid = thepo.source 
525              if duplicatestyle == "msgid_comment_all": 
526                  addcomment(thepo) 
527                  uniqueunits.append(thepo) 
528              elif msgid in msgiddict: 
529                  if duplicatestyle == "merge": 
530                      if msgid: 
531                          msgiddict[msgid].merge(thepo) 
532                      else: 
533                          addcomment(thepo) 
534                          uniqueunits.append(thepo) 
535                  elif duplicatestyle == "keep": 
536                      uniqueunits.append(thepo) 
537                  elif duplicatestyle == "msgid_comment": 
538                      origpo = msgiddict[msgid] 
539                      if origpo not in markedpos: 
540                          addcomment(origpo) 
541                      addcomment(thepo) 
542                      uniqueunits.append(thepo) 
543                  elif duplicatestyle == "msgctxt": 
544                      origpo = msgiddict[msgid] 
545                      if origpo not in markedpos: 
546                          gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join(origpo.getlocations())) 
547                          markedpos.append(thepo) 
548                      gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations())) 
549                      uniqueunits.append(thepo) 
550              else: 
551                  if not msgid and duplicatestyle != "keep": 
552                      addcomment(thepo) 
553                  msgiddict[msgid] = thepo 
554                  uniqueunits.append(thepo) 
555          new_gpo_memory_file = gpo.po_file_create() 
556          new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None) 
557          for unit in uniqueunits: 
558              gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message) 
559          gpo.po_message_iterator_free(self._gpo_message_iterator) 
560          self._gpo_message_iterator = new_gpo_message_iterator 
561          self._gpo_memory_file = new_gpo_memory_file 
562          self.units = uniqueunits 
 563   
565          def obsolete_workaround(): 
566               
567               
568               
569              for unit in self.units: 
570                  if unit.isobsolete(): 
571                      gpo.po_message_set_extracted_comments(unit._gpo_message, "") 
572                      location = gpo.po_message_filepos(unit._gpo_message, 0) 
573                      while location: 
574                          gpo.po_message_remove_filepos(unit._gpo_message, 0) 
575                          location = gpo.po_message_filepos(unit._gpo_message, 0) 
 576          outputstring = "" 
577          if self._gpo_memory_file: 
578              obsolete_workaround() 
579              outputfile = os.tmpnam() 
580              f = open(outputfile, "w") 
581              self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, outputfile, xerror_handler) 
582              f.close() 
583              f = open(outputfile, "r") 
584              outputstring = f.read() 
585              f.close() 
586              os.remove(outputfile) 
587          return outputstring 
588   
590          """Returns True if the object doesn't contain any translation units.""" 
591          if len(self.units) == 0: 
592              return True 
593           
594          if self.units[0].isheader(): 
595              units = self.units[1:] 
596          else: 
597              units = self.units 
598   
599          for unit in units: 
600              if not unit.isblank() and not unit.isobsolete(): 
601                  return False 
602          return True 
 603   
605          if hasattr(input, 'name'): 
606              self.filename = input.name 
607          elif not getattr(self, 'filename', ''): 
608              self.filename = '' 
609          if hasattr(input, "read"): 
610              posrc = input.read() 
611              input.close() 
612              input = posrc 
613          needtmpfile = not os.path.isfile(input) 
614          if needtmpfile: 
615               
616              tmpfile = os.tmpnam() 
617              f = open(tmpfile, "w") 
618              f.write(input) 
619              f.close() 
620              input = tmpfile 
621          self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler) 
622          if self._gpo_memory_file is None: 
623              print >> sys.stderr, "Error:" 
624          if needtmpfile: 
625              os.remove(tmpfile) 
626           
627          self._header = gpo.po_file_domain_header(self._gpo_memory_file, None) 
628          if self._header: 
629              charset = gpo.po_header_field(self._header, "Content-Type") 
630              if charset: 
631                  charset = re.search("charset=([^\\s]+)", charset).group(1) 
632              self._encoding = encodingToUse(charset) 
633          self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None) 
634          newmessage = gpo.po_next_message(self._gpo_message_iterator) 
635          while newmessage: 
636              newunit = pounit(gpo_message=newmessage) 
637              self.units.append(newunit) 
638              newmessage = gpo.po_next_message(self._gpo_message_iterator) 
639          self._free_iterator() 
 640   
642           
643           
644          return 
645          self._free_iterator() 
646          if self._gpo_memory_file is not None: 
647              gpo.po_file_free(self._gpo_memory_file) 
648              self._gpo_memory_file = None 
 649   
651           
652           
653          return 
654          if self._gpo_message_iterator is not None: 
655              gpo.po_message_iterator_free(self._gpo_message_iterator) 
656              self._gpo_message_iterator = None 
 657