1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """Classes that hold units of .po files (pounit) or entire files (pofile).
23
24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and
25 many other projects.
26
27 This uses libgettextpo from the gettext package. Any version before 0.17 will
28 at least cause some subtle bugs or may not work at all. Developers might want
29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext
30 package for the public API of the library.
31 """
32
33 from translate.misc.multistring import multistring
34 from translate.storage import pocommon
35 from translate.storage import pypo
36 from translate.storage.pocommon import encodingToUse
37 from translate.lang import data
38 from ctypes import c_int, c_uint, c_char_p, c_long, CFUNCTYPE, POINTER
39 from ctypes import Structure, cdll
40 import ctypes.util
41 import os
42 import re
43 import sys
44 import tempfile
45
46 lsep = " "
47 """Seperator for #: entries"""
48
49 STRING = c_char_p
50
51
52
55
56
57 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
58 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
59
60
61
65
66
68 _fields_ = [
69 ('error', CFUNCTYPE(None, c_int, c_int, STRING)),
70 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)),
71 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)),
72 ('multiline_error', CFUNCTYPE(None, STRING, STRING)),
73 ]
74
75
76
77 -def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_text):
78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, column, multilint_p, message_text
79 if severity >= 1:
80 raise ValueError(message_text)
81
82
83 -def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2):
84 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2
85 if severity >= 1:
86 raise ValueError(message_text1)
87
88
89
90 gpo = None
91
92
93 names = ['gettextpo', 'libgettextpo']
94 for name in names:
95 lib_location = ctypes.util.find_library(name)
96 if lib_location:
97 gpo = cdll.LoadLibrary(lib_location)
98 if gpo:
99 break
100 else:
101
102
103 try:
104 gpo = cdll.LoadLibrary('libgettextpo.so')
105 except OSError, e:
106 raise ImportError("gettext PO library not found")
107
108
109
110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)]
111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)]
112 gpo.po_file_write_v2.retype = c_int
113
114
115 gpo.po_file_domain_header.restype = STRING
116 gpo.po_header_field.restype = STRING
117 gpo.po_header_field.argtypes = [STRING, STRING]
118
119
120 gpo.po_filepos_file.restype = STRING
121 gpo.po_message_filepos.restype = c_int
122 gpo.po_message_filepos.argtypes = [c_int, c_int]
123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int]
124
125
126 gpo.po_message_comments.restype = STRING
127 gpo.po_message_extracted_comments.restype = STRING
128 gpo.po_message_prev_msgctxt.restype = STRING
129 gpo.po_message_prev_msgid.restype = STRING
130 gpo.po_message_prev_msgid_plural.restype = STRING
131 gpo.po_message_is_format.restype = c_int
132 gpo.po_message_is_format.argtypes = [c_int, STRING]
133 gpo.po_message_set_format.argtypes = [c_int, STRING, c_int]
134 gpo.po_message_msgctxt.restype = STRING
135 gpo.po_message_msgid.restype = STRING
136 gpo.po_message_msgid_plural.restype = STRING
137 gpo.po_message_msgstr.restype = STRING
138 gpo.po_message_msgstr_plural.restype = STRING
139
140
141 gpo.po_message_set_comments.argtypes = [c_int, STRING]
142 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING]
143 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int]
144 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING]
145
146
147 xerror_handler = po_xerror_handler()
148 xerror_handler.xerror = xerror_prototype(xerror_cb)
149 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb)
150
151
154
155
158
159
162
163
165 """Returns the libgettextpo version
166
167 @rtype: three-value tuple
168 @return: libgettextpo version in the following format::
169 (major version, minor version, subminor version)
170 """
171 libversion = c_long.in_dll(gpo, 'libgettextpo_version')
172 major = libversion.value >> 16
173 minor = libversion.value >> 8
174 subminor = libversion.value - (major << 16) - (minor << 8)
175 return major, minor, subminor
176
177
178 -class pounit(pocommon.pounit):
179
180 - def __init__(self, source=None, encoding='utf-8', gpo_message=None):
181 self._rich_source = None
182 self._rich_target = None
183 self._encoding = encoding
184 if not gpo_message:
185 self._gpo_message = gpo.po_message_create()
186 if source or source == "":
187 self.source = source
188 self.target = ""
189 elif gpo_message:
190 self._gpo_message = gpo_message
191
196 msgid_plural = property(None, setmsgid_plural)
197
199
200 def remove_msgid_comments(text):
201 if not text:
202 return text
203 if text.startswith("_:"):
204 remainder = re.search(r"_: .*\n(.*)", text)
205 if remainder:
206 return remainder.group(1)
207 else:
208 return u""
209 else:
210 return text
211 singular = remove_msgid_comments(gpo.po_message_msgid(self._gpo_message).decode(self._encoding))
212 if singular:
213 if self.hasplural():
214 multi = multistring(singular, self._encoding)
215 pluralform = gpo.po_message_msgid_plural(self._gpo_message).decode(self._encoding)
216 multi.strings.append(pluralform)
217 return multi
218 else:
219 return singular
220 else:
221 return u""
222
235 source = property(getsource, setsource)
236
238 if self.hasplural():
239 plurals = []
240 nplural = 0
241 plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
242 while plural:
243 plurals.append(plural.decode(self._encoding))
244 nplural += 1
245 plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
246 if plurals:
247 multi = multistring(plurals, encoding=self._encoding)
248 else:
249 multi = multistring(u"")
250 else:
251 multi = (gpo.po_message_msgstr(self._gpo_message) or "").decode(self._encoding)
252 return multi
253
255
256 if self.hasplural():
257 if isinstance(target, multistring):
258 target = target.strings
259 elif isinstance(target, basestring):
260 target = [target]
261
262 elif isinstance(target, (dict, list)):
263 if len(target) == 1:
264 target = target[0]
265 else:
266 raise ValueError("po msgid element has no plural but msgstr has %d elements (%s)" % (len(target), target))
267
268
269
270
271
272 if isinstance(target, (dict, list)):
273 i = 0
274 message = gpo.po_message_msgstr_plural(self._gpo_message, i)
275 while message is not None:
276 gpo.po_message_set_msgstr_plural(self._gpo_message, i, None)
277 i += 1
278 message = gpo.po_message_msgstr_plural(self._gpo_message, i)
279
280 if isinstance(target, list):
281 for i in range(len(target)):
282 targetstring = target[i]
283 if isinstance(targetstring, unicode):
284 targetstring = targetstring.encode(self._encoding)
285 gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
286
287 elif isinstance(target, dict):
288 for i, targetstring in enumerate(target.itervalues()):
289 gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
290
291 else:
292 if isinstance(target, unicode):
293 target = target.encode(self._encoding)
294 if target is None:
295 gpo.po_message_set_msgstr(self._gpo_message, "")
296 else:
297 gpo.po_message_set_msgstr(self._gpo_message, target)
298 target = property(gettarget, settarget)
299
301 """The unique identifier for this unit according to the convensions in
302 .mo files."""
303 id = (gpo.po_message_msgid(self._gpo_message) or "").decode(self._encoding)
304
305
306
307
308
309
310
311 context = gpo.po_message_msgctxt(self._gpo_message)
312 if context:
313 id = u"%s\04%s" % (context.decode(self._encoding), id)
314 return id
315
317 if origin == None:
318 comments = gpo.po_message_comments(self._gpo_message) + \
319 gpo.po_message_extracted_comments(self._gpo_message)
320 elif origin == "translator":
321 comments = gpo.po_message_comments(self._gpo_message)
322 elif origin in ["programmer", "developer", "source code"]:
323 comments = gpo.po_message_extracted_comments(self._gpo_message)
324 else:
325 raise ValueError("Comment type not valid")
326
327 if comments and get_libgettextpo_version() < (0, 17, 0):
328 comments = "\n".join([line.strip() for line in comments.split("\n")])
329
330 return comments[:-1].decode(self._encoding)
331
332 - def addnote(self, text, origin=None, position="append"):
333
334 if not (text and text.strip()):
335 return
336 text = data.forceunicode(text)
337 oldnotes = self.getnotes(origin)
338 newnotes = None
339 if oldnotes:
340 if position == "append":
341 newnotes = oldnotes + "\n" + text
342 elif position == "merge":
343 if oldnotes != text:
344 oldnoteslist = oldnotes.split("\n")
345 for newline in text.split("\n"):
346 newline = newline.rstrip()
347
348 if newline not in oldnotes or len(newline) < 5:
349 oldnoteslist.append(newline)
350 newnotes = "\n".join(oldnoteslist)
351 else:
352 newnotes = text + '\n' + oldnotes
353 else:
354 newnotes = "\n".join([line.rstrip() for line in text.split("\n")])
355
356 if newnotes:
357 newlines = []
358 needs_space = get_libgettextpo_version() < (0, 17, 0)
359 for line in newnotes.split("\n"):
360 if line and needs_space:
361 newlines.append(" " + line)
362 else:
363 newlines.append(line)
364 newnotes = "\n".join(newlines).encode(self._encoding)
365 if origin in ["programmer", "developer", "source code"]:
366 gpo.po_message_set_extracted_comments(self._gpo_message, newnotes)
367 else:
368 gpo.po_message_set_comments(self._gpo_message, newnotes)
369
371 gpo.po_message_set_comments(self._gpo_message, "")
372
374 newpo = self.__class__()
375 newpo._gpo_message = self._gpo_message
376 return newpo
377
378 - def merge(self, otherpo, overwrite=False, comments=True, authoritative=False):
412
414
415
416 return self.getid() == "" and len(self.target) > 0
417
420
423
426
433
435 return gpo.po_message_is_fuzzy(self._gpo_message)
436
438 gpo.po_message_set_fuzzy(self._gpo_message, present)
439
441 return gpo.po_message_is_obsolete(self._gpo_message)
442
444
445
446 gpo.po_message_set_obsolete(self._gpo_message, True)
447
449 gpo.po_message_set_obsolete(self._gpo_message, False)
450
452 return gpo.po_message_msgid_plural(self._gpo_message) is not None
453
465
469 msgidcomment = property(_extract_msgidcomments, setmsgidcomment)
470
475
477 locations = []
478 i = 0
479 location = gpo.po_message_filepos(self._gpo_message, i)
480 while location:
481 locname = gpo.po_filepos_file(location)
482 locline = gpo.po_filepos_start_line(location)
483 if locline == -1:
484 locstring = locname
485 else:
486 locstring = locname + ":" + str(locline)
487 locations.append(locstring)
488 i += 1
489 location = gpo.po_message_filepos(self._gpo_message, i)
490 return locations
491
493 for loc in location.split():
494 parts = loc.split(":")
495 file = parts[0]
496 if len(parts) == 2:
497 line = int(parts[1] or "0")
498 else:
499 line = -1
500 gpo.po_message_add_filepos(self._gpo_message, file, line)
501
502 - def getcontext(self):
503 msgctxt = gpo.po_message_msgctxt(self._gpo_message)
504 if msgctxt:
505 return msgctxt.decode(self._encoding)
506 else:
507 msgidcomment = self._extract_msgidcomments()
508 return msgidcomment
509
544 buildfromunit = classmethod(buildfromunit)
545
546
547 -class pofile(pocommon.pofile):
548 UnitClass = pounit
549
551 self._gpo_memory_file = None
552 self._gpo_message_iterator = None
553 super(pofile, self).__init__(inputfile=inputfile, encoding=encoding)
554 if inputfile is None:
555 self._gpo_memory_file = gpo.po_file_create()
556 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
557
558 - def addunit(self, unit, new=True):
559 if new:
560 gpo.po_message_insert(self._gpo_message_iterator, unit._gpo_message)
561 super(pofile, self).addunit(unit)
562
564 """make sure each msgid is unique ; merge comments etc from duplicates into original"""
565
566
567 id_dict = {}
568 uniqueunits = []
569
570
571 markedpos = []
572 def addcomment(thepo):
573 thepo.msgidcomment = " ".join(thepo.getlocations())
574 markedpos.append(thepo)
575 for thepo in self.units:
576 id = thepo.getid()
577 if thepo.isheader() and not thepo.getlocations():
578
579 uniqueunits.append(thepo)
580 elif id in id_dict:
581 if duplicatestyle == "merge":
582 if id:
583 id_dict[id].merge(thepo)
584 else:
585 addcomment(thepo)
586 uniqueunits.append(thepo)
587 elif duplicatestyle == "msgctxt":
588 origpo = id_dict[id]
589 if origpo not in markedpos:
590 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join(origpo.getlocations()))
591 markedpos.append(thepo)
592 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations()))
593 uniqueunits.append(thepo)
594 else:
595 if not id:
596 if duplicatestyle == "merge":
597 addcomment(thepo)
598 else:
599 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations()))
600 id_dict[id] = thepo
601 uniqueunits.append(thepo)
602 new_gpo_memory_file = gpo.po_file_create()
603 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None)
604 for unit in uniqueunits:
605 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message)
606 gpo.po_message_iterator_free(self._gpo_message_iterator)
607 self._gpo_message_iterator = new_gpo_message_iterator
608 self._gpo_memory_file = new_gpo_memory_file
609 self.units = uniqueunits
610
612 def obsolete_workaround():
613
614
615
616 for unit in self.units:
617 if unit.isobsolete():
618 gpo.po_message_set_extracted_comments(unit._gpo_message, "")
619 location = gpo.po_message_filepos(unit._gpo_message, 0)
620 while location:
621 gpo.po_message_remove_filepos(unit._gpo_message, 0)
622 location = gpo.po_message_filepos(unit._gpo_message, 0)
623 outputstring = ""
624 if self._gpo_memory_file:
625 obsolete_workaround()
626 f, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
627 os.close(f)
628 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, fname, xerror_handler)
629 f = open(fname)
630 outputstring = f.read()
631 f.close()
632 os.remove(fname)
633 return outputstring
634
636 """Returns True if the object doesn't contain any translation units."""
637 if len(self.units) == 0:
638 return True
639
640 if self.units[0].isheader():
641 units = self.units[1:]
642 else:
643 units = self.units
644
645 for unit in units:
646 if not unit.isblank() and not unit.isobsolete():
647 return False
648 return True
649
651 if hasattr(input, 'name'):
652 self.filename = input.name
653 elif not getattr(self, 'filename', ''):
654 self.filename = ''
655
656 if hasattr(input, "read"):
657 posrc = input.read()
658 input.close()
659 input = posrc
660
661 needtmpfile = not os.path.isfile(input)
662 if needtmpfile:
663
664 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
665 os.write(fd, input)
666 input = fname
667 os.close(fd)
668
669 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler)
670 if self._gpo_memory_file is None:
671 print >> sys.stderr, "Error:"
672
673 if needtmpfile:
674 os.remove(input)
675
676
677 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None)
678 if self._header:
679 charset = gpo.po_header_field(self._header, "Content-Type")
680 if charset:
681 charset = re.search("charset=([^\\s]+)", charset).group(1)
682 self._encoding = encodingToUse(charset)
683 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
684 newmessage = gpo.po_next_message(self._gpo_message_iterator)
685 while newmessage:
686 newunit = pounit(gpo_message=newmessage, encoding=self._encoding)
687 self.addunit(newunit, new=False)
688 newmessage = gpo.po_next_message(self._gpo_message_iterator)
689 self._free_iterator()
690
692
693
694 return
695 self._free_iterator()
696 if self._gpo_memory_file is not None:
697 gpo.po_file_free(self._gpo_memory_file)
698 self._gpo_memory_file = None
699
701
702
703 return
704 if self._gpo_message_iterator is not None:
705 gpo.po_message_iterator_free(self._gpo_message_iterator)
706 self._gpo_message_iterator = None
707