From c1607cb3ad1bac02abd844516ce2dc052fd0cc12 Mon Sep 17 00:00:00 2001 From: jvoisin Date: Tue, 16 Aug 2011 18:15:57 +0200 Subject: Add fogottent stuff --- MANIFEST.in | 2 +- README | 29 +- mat/bencode/__init__.py | 1 + mat/bencode/bencode.py | 142 ++ mat/hachoir_editor/__init__.py | 8 + mat/hachoir_editor/field.py | 69 + mat/hachoir_editor/fieldset.py | 347 +++++ mat/hachoir_editor/typed_field.py | 253 ++++ mat/pdfrw/__init__.py | 14 + mat/pdfrw/pdfcompress.py | 57 + mat/pdfrw/pdfobjects.py | 183 +++ mat/pdfrw/pdfreader.py | 213 +++ mat/pdfrw/pdftokens.py | 249 ++++ mat/pdfrw/pdfwriter.py | 234 ++++ mat/tarfile/__init__.py | 1 + mat/tarfile/tarfile.py | 2594 +++++++++++++++++++++++++++++++++++++ 16 files changed, 4392 insertions(+), 4 deletions(-) create mode 100644 mat/bencode/__init__.py create mode 100644 mat/bencode/bencode.py create mode 100644 mat/hachoir_editor/__init__.py create mode 100644 mat/hachoir_editor/field.py create mode 100644 mat/hachoir_editor/fieldset.py create mode 100644 mat/hachoir_editor/typed_field.py create mode 100644 mat/pdfrw/__init__.py create mode 100644 mat/pdfrw/pdfcompress.py create mode 100644 mat/pdfrw/pdfobjects.py create mode 100644 mat/pdfrw/pdfreader.py create mode 100644 mat/pdfrw/pdftokens.py create mode 100644 mat/pdfrw/pdfwriter.py create mode 100644 mat/tarfile/__init__.py create mode 100755 mat/tarfile/tarfile.py diff --git a/MANIFEST.in b/MANIFEST.in index ed5f978..3cdc6b9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include README cli.py gui.py -graft lib +graft mat recursive-include test * diff --git a/README b/README index 2b74d21..80596a0 100644 --- a/README +++ b/README @@ -9,7 +9,7 @@ METADATA AND PRIVACY: Cameras record data about when a picture was taken and what camera was used. Office documents like pdf or Office automatically adds author and company information to documents and spreadsheets. - Maybe you don't want to disclose those informations on the web. + Maybe you don't want to disclose those information on the web. WARNING : Mat only remove metadata from your files, it does not anonymise their @@ -27,7 +27,7 @@ DEPENDENCIES: OPTIONALS DEPENDENCIES: - python-poppler and python-poppler : for pdf support + python-poppler and python-cairo : for pdf support python-mutagen : for massive audio format support @@ -110,6 +110,29 @@ SUPPORTED FORMAT: method : removal of harmful fields is done with mutagen +HOW TO IMPLEMENT NEW FORMATS: + 1. add the format's mimetype to the STRIPPER list in mat.py + 2. inherit the GenericParser class (parser.py) + 3. read the parser.py module + 4. implement at least these three methods: + - is_clean(self) + - remove_all(self) + - get_meta(self) + 5. don't forget to call the do_backup() method when necessary + +ALTERNATIVES AND COMPLEMENTS: +for images: + exiftool (perl) : metadata manipulation + exiv2 (C++) : metadata manipulation + graphicsmagick (a fork from imagemagick) : cli image manipulation + +for pdf: + pdfminer (python) : pdf manipulation + +other tools: + an hexadecimal editor + + LICENSE: This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as @@ -128,7 +151,7 @@ LICENSE: THANKS: Mat would not exist without : - - the Google SUmmer of Code, + - the Google Summer of Code, - the Python language - the amazing (and messy) hachoir library, - poppler and cairo's python bindings, diff --git a/mat/bencode/__init__.py b/mat/bencode/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/mat/bencode/__init__.py @@ -0,0 +1 @@ + diff --git a/mat/bencode/bencode.py b/mat/bencode/bencode.py new file mode 100644 index 0000000..4acf788 --- /dev/null +++ b/mat/bencode/bencode.py @@ -0,0 +1,142 @@ +# The contents of this file are subject to the BitTorrent Open Source License +# Version 1.1 (the License). You may not copy or use this file, in either +# source code or executable form, except in compliance with the License. You +# may obtain a copy of the License at http://www.bittorrent.com/license/. +# +# Software distributed under the License is distributed on an AS IS basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. + +# Written by Petru Paler +# Modified by Julien (jvoisin) Voisin + +''' + A quick (and also nice) lib to bencode/bdecode torrent files +''' + + +import types + + +class BTFailure(Exception): + '''Custom Exception''' + pass + + +class Bencached(object): + '''Custom type : cached string''' + __slots__ = ['bencoded'] + + def __init__(self, string): + self.bencoded = string + + +def decode_int(x, f): + '''decode an int''' + f += 1 + newf = x.index('e', f) + n = int(x[f:newf]) + if x[f] == '-': + if x[f + 1] == '0': + raise ValueError + elif x[f] == '0' and newf != f + 1: + raise ValueError + return (n, newf + 1) + + +def decode_string(x, f): + '''decode a string''' + colon = x.index(':', f) + n = int(x[f:colon]) + if x[f] == '0' and colon != f + 1: + raise ValueError + colon += 1 + return (x[colon:colon + n], colon + n) + + +def decode_list(x, f): + '''decode a list''' + result = [] + f += 1 + while x[f] != 'e': + v, f = DECODE_FUNC[x[f]](x, f) + result.append(v) + return (result, f + 1) + + +def decode_dict(x, f): + '''decode a dict''' + result = {} + f += 1 + while x[f] != 'e': + k, f = decode_string(x, f) + result[k], f = DECODE_FUNC[x[f]](x, f) + return (result, f + 1) + + +def encode_bool(x, r): + '''bencode a boolean''' + if x: + encode_int(1, r) + else: + encode_int(0, r) + + +def encode_int(x, r): + '''bencode an integer/float''' + r.extend(('i', str(x), 'e')) + + +def encode_list(x, r): + '''bencode a list/tuple''' + r.append('l') + [ENCODE_FUNC[type(item)](item, r) for item in x] + r.append('e') + + +def encode_dict(x, result): + '''bencode a dict''' + result.append('d') + ilist = x.items() + ilist.sort() + for k, v in ilist: + result.extend((str(len(k)), ':', k)) + ENCODE_FUNC[type(v)](v, result) + result.append('e') + + +DECODE_FUNC = {} +DECODE_FUNC.update(dict([(str(x), decode_string) for x in xrange(9)])) +DECODE_FUNC['l'] = decode_list +DECODE_FUNC['d'] = decode_dict +DECODE_FUNC['i'] = decode_int + + +ENCODE_FUNC = {} +ENCODE_FUNC[Bencached] = lambda x, r: r.append(x.bencoded) +ENCODE_FUNC[types.IntType] = encode_int +ENCODE_FUNC[types.LongType] = encode_int +ENCODE_FUNC[types.StringType] = lambda x, r: r.extend((str(len(x)), ':', x)) +ENCODE_FUNC[types.ListType] = encode_list +ENCODE_FUNC[types.TupleType] = encode_list +ENCODE_FUNC[types.DictType] = encode_dict +ENCODE_FUNC[types.BooleanType] = encode_bool + + +def bencode(string): + '''bencode $string''' + table = [] + ENCODE_FUNC[type(string)](string, table) + return ''.join(table) + + +def bdecode(string): + '''decode $string''' + try: + result, lenght = DECODE_FUNC[string[0]](string, 0) + except (IndexError, KeyError, ValueError): + raise BTFailure('Not a valid bencoded string') + if lenght != len(string): + raise BTFailure('Invalid bencoded value (data after valid prefix)') + return result diff --git a/mat/hachoir_editor/__init__.py b/mat/hachoir_editor/__init__.py new file mode 100644 index 0000000..1835676 --- /dev/null +++ b/mat/hachoir_editor/__init__.py @@ -0,0 +1,8 @@ +from field import ( + EditorError, FakeField) +from typed_field import ( + EditableField, EditableBits, EditableBytes, + EditableInteger, EditableString, + createEditableField) +from fieldset import EditableFieldSet, NewFieldSet, createEditor + diff --git a/mat/hachoir_editor/field.py b/mat/hachoir_editor/field.py new file mode 100644 index 0000000..6b1efe3 --- /dev/null +++ b/mat/hachoir_editor/field.py @@ -0,0 +1,69 @@ +from hachoir_core.error import HachoirError +from hachoir_core.field import joinPath, MissingField + +class EditorError(HachoirError): + pass + +class FakeField(object): + """ + This class have API looks similar to Field API, but objects don't contain + any value: all values are _computed_ by parent methods. + + Example: FakeField(editor, "abc").size calls editor._getFieldSize("abc"). + """ + is_field_set = False + + def __init__(self, parent, name): + self._parent = parent + self._name = name + + def _getPath(self): + return joinPath(self._parent.path, self._name) + path = property(_getPath) + + def _getName(self): + return self._name + name = property(_getName) + + def _getAddress(self): + return self._parent._getFieldAddress(self._name) + address = property(_getAddress) + + def _getSize(self): + return self._parent.input[self._name].size + size = property(_getSize) + + def _getValue(self): + return self._parent.input[self._name].value + value = property(_getValue) + + def createDisplay(self): + # TODO: Returns new value if field is altered + return self._parent.input[self._name].display + display = property(createDisplay) + + def _getParent(self): + return self._parent + parent = property(_getParent) + + def hasValue(self): + return self._parent.input[self._name].hasValue() + + def __getitem__(self, key): + # TODO: Implement this function! + raise MissingField(self, key) + + def _isAltered(self): + return False + is_altered = property(_isAltered) + + def writeInto(self, output): + size = self.size + addr = self._parent._getFieldInputAddress(self._name) + input = self._parent.input + stream = input.stream + if size % 8: + output.copyBitsFrom(stream, addr, size, input.endian) + else: + output.copyBytesFrom(stream, addr, size//8) + diff --git a/mat/hachoir_editor/fieldset.py b/mat/hachoir_editor/fieldset.py new file mode 100644 index 0000000..cbc12f9 --- /dev/null +++ b/mat/hachoir_editor/fieldset.py @@ -0,0 +1,347 @@ +from hachoir_core.dict import UniqKeyError +from hachoir_core.field import MissingField, Float32, Float64, FakeArray +from hachoir_core.compatibility import any +from hachoir_core.i18n import _ +from typed_field import createEditableField +from field import EditorError +from collections import deque # Python 2.4 +import weakref # Python 2.1 +import struct + +class EditableFieldSet(object): + MAX_SIZE = (1 << 40) # Arbitrary limit to catch errors + is_field_set = True + + def __init__(self, parent, fieldset): + self._parent = parent + self.input = fieldset # original FieldSet + self._fields = {} # cache of editable fields + self._deleted = set() # Names of deleted fields + self._inserted = {} # Inserted field (name => list of field, + # where name is the name after) + + def array(self, key): + # FIXME: Use cache? + return FakeArray(self, key) + + def _getParent(self): + return self._parent + parent = property(_getParent) + + def _isAltered(self): + if self._inserted: + return True + if self._deleted: + return True + return any(field.is_altered for field in self._fields.itervalues()) + is_altered = property(_isAltered) + + def reset(self): + """ + Reset the field set and the input field set. + """ + for key, field in self._fields.iteritems(): + if not field.is_altered: + del self._fields[key] + self.input.reset() + + def __len__(self): + return len(self.input) \ + - len(self._deleted) \ + + sum( len(new) for new in self._inserted.itervalues() ) + + def __iter__(self): + for field in self.input: + name = field.name + if name in self._inserted: + for newfield in self._inserted[name]: + yield weakref.proxy(newfield) + if name not in self._deleted: + yield self[name] + if None in self._inserted: + for newfield in self._inserted[None]: + yield weakref.proxy(newfield) + + def insertBefore(self, name, *new_fields): + self._insert(name, new_fields, False) + + def insertAfter(self, name, *new_fields): + self._insert(name, new_fields, True) + + def insert(self, *new_fields): + self._insert(None, new_fields, True) + + def _insert(self, key, new_fields, next): + """ + key is the name of the field before which new_fields + will be inserted. If next is True, the fields will be inserted + _after_ this field. + """ + # Set unique field name + for field in new_fields: + if field._name.endswith("[]"): + self.input.setUniqueFieldName(field) + + # Check that there is no duplicate in inserted fields + new_names = list(field.name for field in new_fields) + names_set = set(new_names) + if len(names_set) != len(new_fields): + duplicates = (name for name in names_set if 1 < new_names.count(name)) + raise UniqKeyError(_("Duplicates in inserted fields: %s") % ", ".join(duplicates)) + + # Check that field names are not in input + if self.input: # Write special version for NewFieldSet? + for name in new_names: + if name in self.input and name not in self._deleted: + raise UniqKeyError(_("Field name '%s' already exists") % name) + + # Check that field names are not in inserted fields + for fields in self._inserted.itervalues(): + for field in fields: + if field.name in new_names: + raise UniqKeyError(_("Field name '%s' already exists") % field.name) + + # Input have already inserted field? + if key in self._inserted: + if next: + self._inserted[key].extend( reversed(new_fields) ) + else: + self._inserted[key].extendleft( reversed(new_fields) ) + return + + # Whould like to insert in inserted fields? + if key: + for fields in self._inserted.itervalues(): + names = [item.name for item in fields] + try: + pos = names.index(key) + except ValueError: + continue + if 0 <= pos: + if next: + pos += 1 + fields.rotate(-pos) + fields.extendleft( reversed(new_fields) ) + fields.rotate(pos) + return + + # Get next field. Use None if we are at the end. + if next: + index = self.input[key].index + 1 + try: + key = self.input[index].name + except IndexError: + key = None + + # Check that field names are not in input + if key not in self.input: + raise MissingField(self, key) + + # Insert in original input + self._inserted[key]= deque(new_fields) + + def _getDescription(self): + return self.input.description + description = property(_getDescription) + + def _getStream(self): + # FIXME: This property is maybe a bad idea since address may be differents + return self.input.stream + stream = property(_getStream) + + def _getName(self): + return self.input.name + name = property(_getName) + + def _getEndian(self): + return self.input.endian + endian = property(_getEndian) + + def _getAddress(self): + if self._parent: + return self._parent._getFieldAddress(self.name) + else: + return 0 + address = property(_getAddress) + + def _getAbsoluteAddress(self): + address = self.address + current = self._parent + while current: + address += current.address + current = current._parent + return address + absolute_address = property(_getAbsoluteAddress) + + def hasValue(self): + return False +# return self._parent.input[self.name].hasValue() + + def _getSize(self): + if self.is_altered: + return sum(field.size for field in self) + else: + return self.input.size + size = property(_getSize) + + def _getPath(self): + return self.input.path + path = property(_getPath) + + def _getOriginalField(self, name): + assert name in self.input + return self.input[name] + + def _getFieldInputAddress(self, name): + """ + Absolute address of a field from the input field set. + """ + assert name in self.input + return self.input[name].absolute_address + + def _getFieldAddress(self, name): + """ + Compute relative address of a field. The operation takes care of + deleted and resized fields. + """ + #assert name not in self._deleted + addr = 0 + for field in self: + if field.name == name: + return addr + addr += field.size + raise MissingField(self, name) + + def _getItemByPath(self, path): + if not path[0]: + path = path[1:] + field = self + for name in path: + field = field[name] + return field + + def __contains__(self, name): + try: + field = self[name] + return (field is not None) + except MissingField: + return False + + def __getitem__(self, key): + """ + Create a weak reference to an editable field (EditableField) for the + field with specified name. If the field is removed later, using the + editable field will raise a weakref.ReferenceError exception. + + May raise a MissingField error if the field doesn't exist in original + field set or it has been deleted. + """ + if "/" in key: + return self._getItemByPath(key.split("/")) + if isinstance(key, (int, long)): + raise EditorError("Integer index are not supported") + + if (key in self._deleted) or (key not in self.input): + raise MissingField(self, key) + if key not in self._fields: + field = self.input[key] + if field.is_field_set: + self._fields[key] = createEditableFieldSet(self, field) + else: + self._fields[key] = createEditableField(self, field) + return weakref.proxy(self._fields[key]) + + def __delitem__(self, name): + """ + Remove a field from the field set. May raise an MissingField exception + if the field has already been deleted. + """ + if name in self._deleted: + raise MissingField(self, name) + self._deleted.add(name) + if name in self._fields: + del self._fields[name] + + def writeInto(self, output): + """ + Write the content if this field set into the output stream + (OutputStream). + """ + if not self.is_altered: + # Not altered: just copy bits/bytes + input = self.input + if input.size % 8: + output.copyBitsFrom(input.stream, + input.absolute_address, input.size, input.endian) + else: + output.copyBytesFrom(input.stream, + input.absolute_address, input.size//8) + else: + # Altered: call writeInto() method of each field + realaddr = 0 + for field in self: + field.writeInto(output) + realaddr += field.size + + def _getValue(self): + raise EditorError('Field set "%s" has no value' % self.path) + def _setValue(self, value): + raise EditorError('Field set "%s" value is read only' % self.path) + value = property(_getValue, _setValue, "Value of field") + +class EditableFloat(EditableFieldSet): + _value = None + + def _isAltered(self): + return (self._value is not None) + is_altered = property(_isAltered) + + def writeInto(self, output): + if self._value is not None: + self._write(output) + else: + EditableFieldSet.writeInto(self, output) + + def _write(self, output): + format = self.input.struct_format + raw = struct.pack(format, self._value) + output.writeBytes(raw) + + def _setValue(self, value): + self.parent._is_altered = True + self._value = value + value = property(EditableFieldSet._getValue, _setValue) + +def createEditableFieldSet(parent, field): + cls = field.__class__ + # FIXME: Support Float80 + if cls in (Float32, Float64): + return EditableFloat(parent, field) + else: + return EditableFieldSet(parent, field) + +class NewFieldSet(EditableFieldSet): + def __init__(self, parent, name): + EditableFieldSet.__init__(self, parent, None) + self._name = name + self._endian = parent.endian + + def __iter__(self): + if None in self._inserted: + return iter(self._inserted[None]) + else: + raise StopIteration() + + def _getName(self): + return self._name + name = property(_getName) + + def _getEndian(self): + return self._endian + endian = property(_getEndian) + + is_altered = property(lambda self: True) + +def createEditor(fieldset): + return EditableFieldSet(None, fieldset) + diff --git a/mat/hachoir_editor/typed_field.py b/mat/hachoir_editor/typed_field.py new file mode 100644 index 0000000..0f0427b --- /dev/null +++ b/mat/hachoir_editor/typed_field.py @@ -0,0 +1,253 @@ +from hachoir_core.field import ( + RawBits, Bit, Bits, PaddingBits, + RawBytes, Bytes, PaddingBytes, + GenericString, Character, + isInteger, isString) +from field import FakeField + +class EditableField(FakeField): + """ + Pure virtual class used to write editable field class. + """ + + _is_altered = False + def __init__(self, parent, name, value=None): + FakeField.__init__(self, parent, name) + self._value = value + + def _isAltered(self): + return self._is_altered + is_altered = property(_isAltered) + + def hasValue(self): + return True + + def _computeSize(self): + raise NotImplementedError() + def _getValue(self): + return self._value + def _setValue(self, value): + self._value = value + + def _propGetValue(self): + if self._value is not None: + return self._getValue() + else: + return FakeField._getValue(self) + def _propSetValue(self, value): + self._setValue(value) + self._is_altered = True + value = property(_propGetValue, _propSetValue) + + def _getSize(self): + if self._value is not None: + return self._computeSize() + else: + return FakeField._getSize(self) + size = property(_getSize) + + def _write(self, output): + raise NotImplementedError() + + def writeInto(self, output): + if self._is_altered: + self._write(output) + else: + return FakeField.writeInto(self, output) + +class EditableFixedField(EditableField): + """ + Editable field with fixed size. + """ + + def __init__(self, parent, name, value=None, size=None): + EditableField.__init__(self, parent, name, value) + if size is not None: + self._size = size + else: + self._size = self._parent._getOriginalField(self._name).size + + def _getSize(self): + return self._size + size = property(_getSize) + +class EditableBits(EditableFixedField): + def __init__(self, parent, name, *args): + if args: + if len(args) != 2: + raise TypeError( + "Wrong argument count, EditableBits constructor prototype is: " + "(parent, name, [size, value])") + size = args[0] + value = args[1] + assert isinstance(value, (int, long)) + else: + size = None + value = None + EditableFixedField.__init__(self, parent, name, value, size) + if args: + self._setValue(args[1]) + self._is_altered = True + + def _setValue(self, value): + if not(0 <= value < (1 << self._size)): + raise ValueError("Invalid value, must be in range %s..%s" + % (0, (1 << self._size) - 1)) + self._value = value + + def _write(self, output): + output.writeBits(self._size, self._value, self._parent.endian) + +class EditableBytes(EditableField): + def _setValue(self, value): + if not value: raise ValueError( + "Unable to set empty string to a EditableBytes field") + self._value = value + + def _computeSize(self): + return len(self._value) * 8 + + def _write(self, output): + output.writeBytes(self._value) + +class EditableString(EditableField): + MAX_SIZE = { + "Pascal8": (1 << 8)-1, + "Pascal16": (1 << 16)-1, + "Pascal32": (1 << 32)-1, + } + + def __init__(self, parent, name, *args, **kw): + if len(args) == 2: + value = args[1] + assert isinstance(value, str) # TODO: support Unicode + elif not args: + value = None + else: + raise TypeError( + "Wrong argument count, EditableString constructor prototype is:" + "(parent, name, [format, value])") + EditableField.__init__(self, parent, name, value) + if len(args) == 2: + self._charset = kw.get('charset', None) + self._format = args[0] + if self._format in GenericString.PASCAL_FORMATS: + self._prefix_size = GenericString.PASCAL_FORMATS[self._format] + else: + self._prefix_size = 0 + self._suffix_str = GenericString.staticSuffixStr( + self._format, self._charset, self._parent.endian) + self._is_altered = True + else: + orig = self._parent._getOriginalField(name) + self._charset = orig.charset + self._format = orig.format + self._prefix_size = orig.content_offset + self._suffix_str = orig.suffix_str + + def _setValue(self, value): + size = len(value) + if self._format in self.MAX_SIZE and self.MAX_SIZE[self._format] < size: + raise ValueError("String is too big") + self._value = value + + def _computeSize(self): + return (self._prefix_size + len(self._value) + len(self._suffix_str))*8 + + def _write(self, output): + if self._format in GenericString.SUFFIX_FORMAT: + output.writeBytes(self._value) + output.writeBytes(self._suffix_str) + elif self._format == "fixed": + output.writeBytes(self._value) + else: + assert self._format in GenericString.PASCAL_FORMATS + size = GenericString.PASCAL_FORMATS[self._format] + output.writeInteger(len(self._value), False, size, self._parent.endian) + output.writeBytes(self._value) + +class EditableCharacter(EditableFixedField): + def __init__(self, parent, name, *args): + if args: + if len(args) != 3: + raise TypeError( + "Wrong argument count, EditableCharacter " + "constructor prototype is: (parent, name, [value])") + value = args[0] + if not isinstance(value, str) or len(value) != 1: + raise TypeError("EditableCharacter needs a character") + else: + value = None + EditableFixedField.__init__(self, parent, name, value, 8) + if args: + self._is_altered = True + + def _setValue(self, value): + if not isinstance(value, str) or len(value) != 1: + raise TypeError("EditableCharacter needs a character") + self._value = value + + def _write(self, output): + output.writeBytes(self._value) + +class EditableInteger(EditableFixedField): + VALID_VALUE_SIGNED = { + 8: (-(1 << 8), (1 << 8)-1), + 16: (-(1 << 15), (1 << 15)-1), + 32: (-(1 << 31), (1 << 31)-1), + } + VALID_VALUE_UNSIGNED = { + 8: (0, (1 << 8)-1), + 16: (0, (1 << 16)-1), + 32: (0, (1 << 32)-1) + } + + def __init__(self, parent, name, *args): + if args: + if len(args) != 3: + raise TypeError( + "Wrong argument count, EditableInteger constructor prototype is: " + "(parent, name, [signed, size, value])") + size = args[1] + value = args[2] + assert isinstance(value, (int, long)) + else: + size = None + value = None + EditableFixedField.__init__(self, parent, name, value, size) + if args: + self._signed = args[0] + self._is_altered = True + else: + self._signed = self._parent._getOriginalField(self._name).signed + + def _setValue(self, value): + if self._signed: + valid = self.VALID_VALUE_SIGNED + else: + valid = self.VALID_VALUE_UNSIGNED + minval, maxval = valid[self._size] + if not(minval <= value <= maxval): + raise ValueError("Invalid value, must be in range %s..%s" + % (minval, maxval)) + self._value = value + + def _write(self, output): + output.writeInteger( + self.value, self._signed, self._size//8, self._parent.endian) + +def createEditableField(fieldset, field): + if isInteger(field): + cls = EditableInteger + elif isString(field): + cls = EditableString + elif field.__class__ in (RawBytes, Bytes, PaddingBytes): + cls = EditableBytes + elif field.__class__ in (RawBits, Bits, Bit, PaddingBits): + cls = EditableBits + elif field.__class__ == Character: + cls = EditableCharacter + else: + cls = FakeField + return cls(fieldset, field.name) + diff --git a/mat/pdfrw/__init__.py b/mat/pdfrw/__init__.py new file mode 100644 index 0000000..26e8c73 --- /dev/null +++ b/mat/pdfrw/__init__.py @@ -0,0 +1,14 @@ +# A part of pdfrw (pdfrw.googlecode.com) +# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas +# MIT license -- See LICENSE.txt for details + +from pdfwriter import PdfWriter +from pdfreader import PdfReader +from pdfobjects import PdfObject, PdfName, PdfArray, PdfDict, IndirectPdfDict, PdfString +from pdftokens import PdfTokens + +# Add a tiny bit of compatibility to pyPdf + +PdfFileReader = PdfReader +PdfFileWriter = PdfWriter + diff --git a/mat/pdfrw/pdfcompress.py b/mat/pdfrw/pdfcompress.py new file mode 100644 index 0000000..1c11970 --- /dev/null +++ b/mat/pdfrw/pdfcompress.py @@ -0,0 +1,57 @@ +# A part of pdfrw (pdfrw.googlecode.com) +# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas +# MIT license -- See LICENSE.txt for details + +''' +Currently, this sad little file only knows how to decompress +using the flate (zlib) algorithm. Maybe more later, but it's +not a priority for me... +''' + +from __future__ import generators + +try: + set +except NameError: + from sets import Set as set + +import zlib +from pdfobjects import PdfDict, PdfName + + +def streamobjects(mylist): + for obj in mylist: + if isinstance(obj, PdfDict) and obj.stream is not None: + yield obj + +def uncompress(mylist, warnings=set()): + flate = PdfName.FlateDecode + for obj in streamobjects(mylist): + ftype = obj.Filter + if ftype is None: + continue + if isinstance(ftype, list) and len(ftype) == 1: + # todo: multiple filters + ftype = ftype[0] + parms = obj.DecodeParms + if ftype != flate or parms is not None: + msg = 'Not decompressing: cannot use filter %s with parameters %s' % (repr(ftype), repr(parms)) + if msg not in warnings: + warnings.add(msg) + print msg + else: + obj.stream = zlib.decompress(obj.stream) + obj.Filter = None + +def compress(mylist): + flate = PdfName.FlateDecode + for obj in streamobjects(mylist): + ftype = obj.Filter + if ftype is not None: + continue + oldstr = obj.stream + newstr = zlib.compress(oldstr) + if len(newstr) < len(oldstr) + 30: + obj.stream = newstr + obj.Filter = flate + obj.DecodeParms = None diff --git a/mat/pdfrw/pdfobjects.py b/mat/pdfrw/pdfobjects.py new file mode 100644 index 0000000..08ad825 --- /dev/null +++ b/mat/pdfrw/pdfobjects.py @@ -0,0 +1,183 @@ +# A part of pdfrw (pdfrw.googlecode.com) +# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas +# MIT license -- See LICENSE.txt for details + +''' +Objects that can occur in PDF files. The most important +objects are arrays and dicts. Either of these can be +indirect or not, and dicts could have an associated +stream. +''' +from __future__ import generators + +try: + set +except NameError: + from sets import Set as set + +import re + +class PdfObject(str): + indirect = False + +class PdfArray(list): + indirect = False + +class PdfName(object): + def __getattr__(self, name): + return self(name) + def __call__(self, name): + return PdfObject('/' + name) + +PdfName = PdfName() + +class PdfString(str): + indirect = False + unescape_dict = {'\\b':'\b', '\\f':'\f', '\\n':'\n', + '\\r':'\r', '\\t':'\t', + '\\\r\n': '', '\\\r':'', '\\\n':'', + '\\\\':'\\', '\\':'', + } + unescape_pattern = r'(\\b|\\f|\\n|\\r|\\t|\\\r\n|\\\r|\\\n|\\[0-9]+|\\)' + unescape_func = re.compile(unescape_pattern).split + + hex_pattern = '([a-fA-F0-9][a-fA-F0-9]|[a-fA-F0-9])' + hex_func = re.compile(hex_pattern).split + + hex_pattern2 = '([a-fA-F0-9][a-fA-F0-9][a-fA-F0-9][a-fA-F0-9]|[a-fA-F0-9][a-fA-F0-9]|[a-fA-F0-9])' + hex_func2 = re.compile(hex_pattern2).split + + hex_funcs = hex_func, hex_func2 + + indirect = False + + def decode_regular(self, remap=chr): + assert self[0] == '(' and self[-1] == ')' + mylist = self.unescape_func(self[1:-1]) + result = [] + unescape = self.unescape_dict.get + for chunk in mylist: + chunk = unescape(chunk, chunk) + if chunk.startswith('\\') and len(chunk) > 1: + value = int(chunk[1:], 8) + # FIXME: TODO: Handle unicode here + if value > 127: + value = 127 + chunk = remap(value) + if chunk: + result.append(chunk) + return ''.join(result) + + def decode_hex(self, remap=chr, twobytes=False): + data = ''.join(self.split()) + data = self.hex_funcs[twobytes](data) + chars = data[1::2] + other = data[0::2] + assert other[0] == '<' and other[-1] == '>' and ''.join(other) == '<>', self + return ''.join([remap(int(x, 16)) for x in chars]) + + def decode(self, remap=chr, twobytes=False): + if self.startswith('('): + return self.decode_regular(remap) + + else: + return self.decode_hex(remap, twobytes) + + def encode(cls, source, usehex=False): + assert not usehex, "Not supported yet" + if isinstance(source, unicode): + source = source.encode('utf-8') + else: + source = str(source) + source = source.replace('\\', '\\\\') + source = source.replace('(', '\\(') + source = source.replace(')', '\\)') + return cls('(' +source + ')') + encode = classmethod(encode) + +class PdfDict(dict): + indirect = False + stream = None + + _special = dict(indirect = ('indirect', False), + stream = ('stream', True), + _stream = ('stream', False), + ) + + def __setitem__(self, name, value): + assert name.startswith('/'), name + if value is not None: + dict.__setitem__(self, name, value) + elif name in self: + del self[name] + + def __init__(self, *args, **kw): + if args: + if len(args) == 1: + args = args[0] + self.update(args) + if isinstance(args, PdfDict): + self.indirect = args.indirect + self._stream = args.stream + for key, value in kw.iteritems(): + setattr(self, key, value) + + def __getattr__(self, name): + return self.get(PdfName(name)) + + def __setattr__(self, name, value): + info = self._special.get(name) + if info is None: + self[PdfName(name)] = value + else: + name, setlen = info + self.__dict__[name] = value + if setlen: + notnone = value is not None + self.Length = notnone and PdfObject(len(value)) or None + + def iteritems(self): + for key, value in dict.iteritems(self): + if value is not None: + assert key.startswith('/'), (key, value) + yield key, value + + def inheritable(self): + ''' Search through ancestors as needed for inheritable + dictionary items + ''' + class Search(object): + def __init__(self, basedict): + self.basedict = basedict + def __getattr__(self, name): + return self[name] + def __getitem__(self, name): + visited = set() + mydict = self.basedict + while 1: + value = getattr(mydict, name) + if value is not None: + return value + myid = id(mydict) + assert myid not in visited + visited.add(myid) + mydict = mydict.Parent + if mydict is None: + return + return Search(self) + inheritable = property(inheritable) + + def private(self): + ''' Allows setting private metadata for use in + processing (not sent to PDF file) + ''' + class Private(object): + pass + + result = Private() + result.__dict__ = self.__dict__ + return result + private = property(private) + +class IndirectPdfDict(PdfDict): + indirect = True diff --git a/mat/pdfrw/pdfreader.py b/mat/pdfrw/pdfreader.py new file mode 100644 index 0000000..6f57bea --- /dev/null +++ b/mat/pdfrw/pdfreader.py @@ -0,0 +1,213 @@ +# A part of pdfrw (pdfrw.googlecode.com) +# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas +# MIT license -- See LICENSE.txt for details + +''' +The PdfReader class reads an entire PDF file into memory and +parses the top-level container objects. (It does not parse +into streams.) The object subclasses PdfDict, and the +document pages are stored in a list in the pages attribute +of the object. +''' + +from pdftokens import PdfTokens +from pdfobjects import PdfDict, PdfArray, PdfName +from pdfcompress import uncompress + +class PdfReader(PdfDict): + + class unresolved: + # Used as a placeholder until we have an object. + pass + + def readindirect(self, objnum, gennum): + ''' Read an indirect object. If it has already + been read, return it from the cache. + ''' + + def setobj(obj): + # Store the new object in the dictionary + # once we have its value + record[1] = obj + + def ordinary(source, setobj, obj): + # Deal with an ordinary (non-array, non-dict) object + setobj(obj) + return obj + + fdata, objnum, gennum = self.fdata, int(objnum), int(gennum) + record = self.indirect_objects[fdata, objnum, gennum] + if record[1] is not self.unresolved: + return record[1] + + # Read the object header and validate it + source = PdfTokens(fdata, record[0]) + objid = source.multiple(3) + assert int(objid[0]) == objnum, objid + assert int(objid[1]) == gennum, objid + assert objid[2] == 'obj', objid + + # Read the object, and call special code if it starts + # an array or dictionary + obj = source.next() + obj = self.special.get(obj, ordinary)(source, setobj, obj) + self.readstream(obj, source) + obj.indirect = True + return obj + + def readstream(obj, source): + ''' Read optional stream following a dictionary + object. + ''' + tok = source.next() + if tok == 'endobj': + return # No stream + + assert isinstance(obj, PdfDict) + assert tok == 'stream', tok + fdata = source.fdata + floc = fdata.rindex(tok, 0, source.floc) + len(tok) + ch = fdata[floc] + if ch == '\r': + floc += 1 + ch = fdata[floc] + assert ch == '\n' + startstream = floc + 1 + endstream = startstream + int(obj.Length) + obj._stream = fdata[startstream:endstream] + source = PdfTokens(fdata, endstream) + endit = source.multiple(2) + if endit != 'endstream endobj'.split(): + # /Length attribute is broken, try to read stream + # anyway disregarding the specified value + # TODO: issue warning here once we have some kind of + # logging + endstream = fdata.index('endstream', startstream) + if fdata[endstream-2:endstream] == '\r\n': + endstream -= 2 + elif fdata[endstream-1] in ['\n', '\r']: + endstream -= 1 + source = PdfTokens(fdata, endstream) + endit = source.multiple(2) + assert endit == 'endstream endobj'.split() + obj.Length = str(endstream-startstream) + obj._stream = fdata[startstream:endstream] + readstream = staticmethod(readstream) + + def readarray(self, source, setobj=lambda x:None, original=None): + special = self.special + result = PdfArray() + setobj(result) + + for value in source: + if value == ']': + break + if value in special: + value = special[value](source) + elif value == 'R': + generation = result.pop() + value = self.readindirect(result.pop(), generation) + result.append(value) + return result + + def readdict(self, source, setobj=lambda x:None, original=None): + special = self.special + result = PdfDict() + setobj(result) + + tok = source.next() + while tok != '>>': + assert tok.startswith('/'), (tok, source.multiple(10)) + key = tok + value = source.next() + if value in special: + value = special[value](source) + tok = source.next() + else: + tok = source.next() + if value.isdigit() and tok.isdigit(): + assert source.next() == 'R' + value = self.readindirect(value, tok) + tok = source.next() + result[key] = value + + return result + + def readxref(fdata): + startloc = fdata.rindex('startxref') + xrefinfo = list(PdfTokens(fdata, startloc, False)) + assert len(xrefinfo) == 3, xrefinfo + assert xrefinfo[0] == 'startxref', xrefinfo[0] + assert xrefinfo[1].isdigit(), xrefinfo[1] + assert xrefinfo[2].rstrip() == '%%EOF', repr(xrefinfo[2]) + return startloc, PdfTokens(fdata, int(xrefinfo[1])) + readxref = staticmethod(readxref) + + def parsexref(self, source): + tok = source.next() + assert tok == 'xref', tok + while 1: + tok = source.next() + if tok == 'trailer': + break + startobj = int(tok) + for objnum in range(startobj, startobj + int(source.next())): + offset = int(source.next()) + generation = int(source.next()) + if source.next() == 'n': + objid = self.fdata, objnum, generation + objval = [offset, self.unresolved] + self.indirect_objects.setdefault(objid, objval) + + pagename = PdfName.Page + pagesname = PdfName.Pages + + def readpages(self, node): + # PDFs can have arbitrarily nested Pages/Page + # dictionary structures. + if node.Type == self.pagename: + return [node] + assert node.Type == self.pagesname, node.Type + result = [] + for node in node.Kids: + result.extend(self.readpages(node)) + return result + + def __init__(self, fname=None, fdata=None, decompress=True): + + if fname is not None: + assert fdata is None + # Allow reading preexisting streams like pyPdf + if hasattr(fname, 'read'): + fdata = fname.read() + else: + f = open(fname, 'rb') + fdata = f.read() + f.close() + + assert fdata is not None + fdata = fdata.rstrip('\00') + self.private.fdata = fdata + + self.private.indirect_objects = {} + self.private.special = {'<<': self.readdict, '[': self.readarray} + + startloc, source = self.readxref(fdata) + self.parsexref(source) + assert source.next() == '<<' + self.update(self.readdict(source)) + assert source.next() == 'startxref' and source.floc > startloc + self.private.pages = self.readpages(self.Root.Pages) + if decompress: + self.uncompress() + + # For compatibility with pyPdf + self.private.numPages = len(self.pages) + + + # For compatibility with pyPdf + def getPage(self, pagenum): + return self.pages[pagenum] + + def uncompress(self): + uncompress([x[1] for x in self.indirect_objects.itervalues()]) diff --git a/mat/pdfrw/pdftokens.py b/mat/pdfrw/pdftokens.py new file mode 100644 index 0000000..04bd559 --- /dev/null +++ b/mat/pdfrw/pdftokens.py @@ -0,0 +1,249 @@ +# A part of pdfrw (pdfrw.googlecode.com) +# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas +# MIT license -- See LICENSE.txt for details + +''' +A tokenizer for PDF streams. + +In general, documentation used was "PDF reference", +sixth edition, for PDF version 1.7, dated November 2006. + +''' + +from __future__ import generators + +try: + set +except NameError: + from sets import Set as set + +import re +from pdfobjects import PdfString, PdfObject + +class _PrimitiveTokens(object): + + # Table 3.1, page 50 of reference, defines whitespace + whitespaceset = set('\x00\t\n\f\r ') + + + # Text on page 50 defines delimiter characters + delimiterset = set('()<>{}[]/%') + + # Coalesce contiguous whitespace into a single token + whitespace_pattern = '[%s]+' % ''.join(whitespaceset) + + # In addition to the delimiters, we also use '\', which + # is special in some contexts in PDF. + delimiter_pattern = '\\\\|\\' + '|\\'.join(delimiterset) + + # Dictionary delimiters are '<<' and '>>'. Look for + # these before the single variety. + dictdelim_pattern = r'\<\<|\>\>' + + pattern = '(%s|%s|%s)' % (whitespace_pattern, + dictdelim_pattern, delimiter_pattern) + re_func = re.compile(pattern).finditer + del whitespace_pattern, dictdelim_pattern + del delimiter_pattern, pattern + + def __init__(self, fdata): + + class MyIterator(object): + def next(): + if not tokens: + startloc = self.startloc + for match in next_match[0]: + start = match.start() + end = match.end() + tappend(fdata[start:end]) + if start > startloc: + tappend(fdata[startloc:start]) + self.startloc = end + break + else: + s = fdata[startloc:] + self.startloc = len(fdata) + if s: + tappend(s) + if not tokens: + raise StopIteration + return tpop() + next = staticmethod(next) + + self.fdata = fdata + self.tokens = tokens = [] + self.iterator = iterator = MyIterator() + self.next = iterator.next + self.next_match = next_match = [None] + tappend = tokens.append + tpop = tokens.pop + + def setstart(self, startloc): + self.startloc = startloc + self.next_match[0] = self.re_func(self.fdata, startloc) + + def __iter__(self): + return self.iterator + + def coalesce(self, result): + ''' This function coalesces tokens together up until + the next delimiter or whitespace. + All of the coalesced tokens will either be non-matches, + or will be a matched backslash. We distinguish the + non-matches by the fact that next() will have left + a following match inside self.tokens for the actual match. + ''' + tokens = self.tokens + whitespace = self.whitespaceset + + # Optimized path for usual case -- regular data (not a name string), + # with no escape character, and followed by whitespace. + + if tokens: + token = tokens.pop() + if token != '\\': + if token[0] not in whitespace: + tokens.append(token) + return + result.append(token) + + # Non-optimized path. Either start of a name string received, + # or we just had one escape. + + for token in self: + if tokens: + result.append(token) + token = tokens.pop() + if token != '\\': + if token[0] not in whitespace: + tokens.append(token) + return + result.append(token) + + + def floc(self): + return self.startloc - sum([len(x) for x in self.tokens]) + +class PdfTokens(object): + + def __init__(self, fdata, startloc=0, strip_comments=True): + + def comment(token): + tokens = [token] + for token in primitive: + tokens.append(token) + if token[0] in whitespaceset and ('\n' in token or '\r' in token): + break + return not strip_comments and ''.join(tokens) + + def single(token): + return token + + def regular_string(token): + def escaped(): + escaped = False + i = -2 + while tokens[i] == '\\': + escaped = not escaped + i -= 1 + return escaped + + tokens = [token] + nestlevel = 1 + for token in primitive: + tokens.append(token) + if token in '()' and not escaped(): + nestlevel += token == '(' or -1 + if not nestlevel: + break + else: + assert 0, "Unexpected end of token stream" + return PdfString(''.join(tokens)) + + def hex_string(token): + tokens = [token] + for token in primitive: + tokens.append(token) + if token == '>': + break + while tokens[-2] == '>>': + tokens.append(tokens.pop(-2)) + return PdfString(''.join(tokens)) + + def normal_data(token): + + # Obscure optimization -- we can get here with + # whitespace or regular character data. If we get + # here with whitespace, then there won't be an additional + # token queued up in the primitive object, otherwise there + # will... + if primitive_tokens: #if token[0] not in whitespaceset: + tokens = [token] + primitive.coalesce(tokens) + return PdfObject(''.join(tokens)) + + def name_string(token): + tokens = [token] + primitive.coalesce(tokens) + token = ''.join(tokens) + if '#' in token: + substrs = token.split('#') + substrs.reverse() + tokens = [substrs.pop()] + while substrs: + s = substrs.pop() + tokens.append(chr(int(s[:2], 16))) + tokens.append(s[2:]) + token = ''.join(tokens) + return PdfObject(token) + + def broken(token): + assert 0, token + + dispatch = { + '(': regular_string, + ')': broken, + '<': hex_string, + '>': broken, + '[': single, + ']': single, + '{': single, + '}': single, + '/': name_string, + '%' : comment, + '<<': single, + '>>': single, + }.get + + class MyIterator(object): + def next(): + while not tokens: + token = primitive_next() + token = dispatch(token, normal_data)(token) + if token: + return token + return tokens.pop() + next = staticmethod(next) + + self.primitive = primitive = _PrimitiveTokens(fdata) + self.setstart = primitive.setstart + primitive.setstart(startloc) + self.fdata = fdata + self.strip_comments = strip_comments + self.tokens = tokens = [] + self.iterator = iterator = MyIterator() + self.next = iterator.next + primitive_next = primitive.next + primitive_tokens = primitive.tokens + whitespaceset = _PrimitiveTokens.whitespaceset + + def floc(self): + return self.primitive.floc() - sum([len(x) for x in self.tokens]) + floc = property(floc) + + def __iter__(self): + return self.iterator + + def multiple(self, count): + next = self.next + return [next() for i in range(count)] diff --git a/mat/pdfrw/pdfwriter.py b/mat/pdfrw/pdfwriter.py new file mode 100644 index 0000000..c193843 --- /dev/null +++ b/mat/pdfrw/pdfwriter.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python + +# A part of pdfrw (pdfrw.googlecode.com) +# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas +# MIT license -- See LICENSE.txt for details + +''' +The PdfWriter class writes an entire PDF file out to disk. + +The writing process is not at all optimized or organized. + +An instance of the PdfWriter class has two methods: + addpage(page) +and + write(fname) + +addpage() assumes that the pages are part of a valid +tree/forest of PDF objects. +''' + +try: + set +except NameError: + from sets import Set as set + +from pdfobjects import PdfName, PdfArray, PdfDict, IndirectPdfDict, PdfObject, PdfString +from pdfcompress import compress + +debug = False + +class FormatObjects(object): + ''' FormatObjects performs the actual formatting and disk write. + ''' + + def add(self, obj, visited): + ''' Add an object to our list, if it's an indirect + object. Just format it if not. + ''' + # Can't hash dicts, so just hash the object ID + objid = id(obj) + + # Automatically set stream objects to indirect + if isinstance(obj, PdfDict): + indirect = obj.indirect or (obj.stream is not None) + else: + indirect = getattr(obj, 'indirect', False) + + if not indirect: + assert objid not in visited, \ + 'Circular reference encountered in non-indirect object %s' % repr(obj) + visited.add(objid) + result = self.format_obj(obj, visited) + visited.remove(objid) + return result + + objnum = self.indirect_dict.get(objid) + + # If we haven't seen the object yet, we need to + # add it to the indirect object list. + if objnum is None: + objlist = self.objlist + objnum = len(objlist) + 1 + if debug: + print ' Object', objnum, '\r', + objlist.append(None) + self.indirect_dict[objid] = objnum + objlist[objnum-1] = self.format_obj(obj) + return '%s 0 R' % objnum + + def format_array(myarray, formatter): + # Format array data into semi-readable ASCII + if sum([len(x) for x in myarray]) <= 70: + return formatter % ' '.join(myarray) + bigarray = [] + count = 1000000 + for x in myarray: + lenx = len(x) + if lenx + count > 70: + subarray = [] + bigarray.append(subarray) + count = 0 + count += lenx + 1 + subarray.append(x) + return formatter % '\n '.join([' '.join(x) for x in bigarray]) + format_array = staticmethod(format_array) + + def format_obj(self, obj, visited=None): + ''' format PDF object data into semi-readable ASCII. + May mutually recurse with add() -- add() will + return references for indirect objects, and add + the indirect object to the list. + ''' + if visited is None: + visited = set() + if isinstance(obj, PdfArray): + myarray = [self.add(x, visited) for x in obj] + return self.format_array(myarray, '[%s]') + elif isinstance(obj, PdfDict): + if self.compress and obj.stream: + compress([obj]) + myarray = [] + # Jython 2.2.1 has a bug which segfaults when + # sorting subclassed strings, so we un-subclass them. + dictkeys = [str(x) for x in obj.iterkeys()] + dictkeys.sort() + for key in dictkeys: + myarray.append(key) + myarray.append(self.add(obj[key], visited)) + result = self.format_array(myarray, '<<%s>>') + stream = obj.stream + if stream is not None: + result = '%s\nstream\n%s\nendstream' % (result, stream) + return result + elif isinstance(obj, basestring) and not hasattr(obj, 'indirect'): + return PdfString.encode(obj) + else: + return str(obj) + + def dump(cls, f, trailer, version='1.3', compress=True): + self = cls() + self.compress = compress + self.indirect_dict = {} + self.objlist = [] + + # The first format of trailer gets all the information, + # but we throw away the actual trailer formatting. + self.format_obj(trailer) + # Now we know the size, so we update the trailer dict + # and get the formatted data. + trailer.Size = PdfObject(len(self.objlist) + 1) + trailer = self.format_obj(trailer) + + # Now we have all the pieces to write out to the file. + # Keep careful track of the counts while we do it so + # we can correctly build the cross-reference. + + header = '%%PDF-%s\n%%\xe2\xe3\xcf\xd3\n' % version + f.write(header) + offset = len(header) + offsets = [(0, 65535, 'f')] + + for i, x in enumerate(self.objlist): + objstr = '%s 0 obj\n%s\nendobj\n' % (i + 1, x) + offsets.append((offset, 0, 'n')) + offset += len(objstr) + f.write(objstr) + + f.write('xref\n0 %s\n' % len(offsets)) + for x in offsets: + f.write('%010d %05d %s\r\n' % x) + f.write('trailer\n\n%s\nstartxref\n%s\n%%%%EOF\n' % (trailer, offset)) + dump = classmethod(dump) + +class PdfWriter(object): + + _trailer = None + + def __init__(self, version='1.3', compress=True): + self.pagearray = PdfArray() + self.compress = compress + self.version = version + + def addpage(self, page): + self._trailer = None + assert page.Type == PdfName.Page + inheritable = page.inheritable # searches for resources + self.pagearray.append( + IndirectPdfDict( + page, + Resources = inheritable.Resources, + MediaBox = inheritable.MediaBox, + CropBox = inheritable.CropBox, + Rotate = inheritable.Rotate, + ) + ) + return self + + addPage = addpage # for compatibility with pyPdf + + def addpages(self, pagelist): + for page in pagelist: + self.addpage(page) + return self + + def _get_trailer(self): + trailer = self._trailer + if trailer is not None: + return trailer + + # Create the basic object structure of the PDF file + trailer = PdfDict( + Root = IndirectPdfDict( + Type = PdfName.Catalog, + Pages = IndirectPdfDict( + Type = PdfName.Pages, + Count = PdfObject(len(self.pagearray)), + Kids = self.pagearray + ) + ) + ) + # Make all the pages point back to the page dictionary + pagedict = trailer.Root.Pages + for page in pagedict.Kids: + page.Parent = pagedict + self._trailer = trailer + return trailer + + def _set_trailer(self, trailer): + self._trailer = trailer + + trailer = property(_get_trailer, _set_trailer) + + def write(self, fname, trailer=None): + trailer = trailer or self.trailer + + # Dump the data. We either have a filename or a preexisting + # file object. + preexisting = hasattr(fname, 'write') + f = preexisting and fname or open(fname, 'wb') + FormatObjects.dump(f, trailer, self.version, self.compress) + if not preexisting: + f.close() + +if __name__ == '__main__': + debug = True + import pdfreader + x = pdfreader.PdfReader('source.pdf') + y = PdfWriter() + for i, page in enumerate(x.pages): + print ' Adding page', i+1, '\r', + y.addpage(page) + print + y.write('result.pdf') + print diff --git a/mat/tarfile/__init__.py b/mat/tarfile/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/mat/tarfile/__init__.py @@ -0,0 +1 @@ + diff --git a/mat/tarfile/tarfile.py b/mat/tarfile/tarfile.py new file mode 100755 index 0000000..37c9b92 --- /dev/null +++ b/mat/tarfile/tarfile.py @@ -0,0 +1,2594 @@ +#! /usr/bin/python2.7 +# -*- coding: iso-8859-1 -*- +#------------------------------------------------------------------- +# tarfile.py +#------------------------------------------------------------------- +# Copyright (C) 2002 Lars Gustäbel +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +"""Read from and write to tar format archives. +""" + +__version__ = "$Revision$" +# $Source$ + +version = "0.9.0" +__author__ = "Lars Gustäbel (lars@gustaebel.de)" +__date__ = "$Date$" +__cvsid__ = "$Id$" +__credits__ = "Gustavo Niemeyer, Niels Gustäbel, Richard Townsend." + +#--------- +# Imports +#--------- +import sys +import os +import shutil +import stat +import errno +import time +import struct +import copy +import re +import operator + +try: + import grp, pwd +except ImportError: + grp = pwd = None + +# from tarfile import * +__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"] + +#--------------------------------------------------------- +# tar constants +#--------------------------------------------------------- +NUL = "\0" # the null character +BLOCKSIZE = 512 # length of processing blocks +RECORDSIZE = BLOCKSIZE * 20 # length of records +GNU_MAGIC = "ustar \0" # magic gnu tar string +POSIX_MAGIC = "ustar\x0000" # magic posix tar string + +LENGTH_NAME = 100 # maximum length of a filename +LENGTH_LINK = 100 # maximum length of a linkname +LENGTH_PREFIX = 155 # maximum length of the prefix field + +REGTYPE = "0" # regular file +AREGTYPE = "\0" # regular file +LNKTYPE = "1" # link (inside tarfile) +SYMTYPE = "2" # symbolic link +CHRTYPE = "3" # character special device +BLKTYPE = "4" # block special device +DIRTYPE = "5" # directory +FIFOTYPE = "6" # fifo special device +CONTTYPE = "7" # contiguous file + +GNUTYPE_LONGNAME = "L" # GNU tar longname +GNUTYPE_LONGLINK = "K" # GNU tar longlink +GNUTYPE_SPARSE = "S" # GNU tar sparse file + +XHDTYPE = "x" # POSIX.1-2001 extended header +XGLTYPE = "g" # POSIX.1-2001 global header +SOLARIS_XHDTYPE = "X" # Solaris extended header + +USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format +GNU_FORMAT = 1 # GNU tar format +PAX_FORMAT = 2 # POSIX.1-2001 (pax) format +DEFAULT_FORMAT = GNU_FORMAT + +#--------------------------------------------------------- +# tarfile constants +#--------------------------------------------------------- +# File types that tarfile supports: +SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE, + SYMTYPE, DIRTYPE, FIFOTYPE, + CONTTYPE, CHRTYPE, BLKTYPE, + GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, + GNUTYPE_SPARSE) + +# File types that will be treated as a regular file. +REGULAR_TYPES = (REGTYPE, AREGTYPE, + CONTTYPE, GNUTYPE_SPARSE) + +# File types that are part of the GNU tar format. +GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, + GNUTYPE_SPARSE) + +# Fields from a pax header that override a TarInfo attribute. +PAX_FIELDS = ("path", "linkpath", "size", "mtime", + "uid", "gid", "uname", "gname") + +# Fields in a pax header that are numbers, all other fields +# are treated as strings. +PAX_NUMBER_FIELDS = { + "atime": float, + "ctime": float, + "mtime": float, + "uid": int, + "gid": int, + "size": int +} + +#--------------------------------------------------------- +# Bits used in the mode field, values in octal. +#--------------------------------------------------------- +S_IFLNK = 0120000 # symbolic link +S_IFREG = 0100000 # regular file +S_IFBLK = 0060000 # block device +S_IFDIR = 0040000 # directory +S_IFCHR = 0020000 # character device +S_IFIFO = 0010000 # fifo + +TSUID = 04000 # set UID on execution +TSGID = 02000 # set GID on execution +TSVTX = 01000 # reserved + +TUREAD = 0400 # read by owner +TUWRITE = 0200 # write by owner +TUEXEC = 0100 # execute/search by owner +TGREAD = 0040 # read by group +TGWRITE = 0020 # write by group +TGEXEC = 0010 # execute/search by group +TOREAD = 0004 # read by other +TOWRITE = 0002 # write by other +TOEXEC = 0001 # execute/search by other + +#--------------------------------------------------------- +# initialization +#--------------------------------------------------------- +ENCODING = sys.getfilesystemencoding() +if ENCODING is None: + ENCODING = sys.getdefaultencoding() + +#--------------------------------------------------------- +# Some useful functions +#--------------------------------------------------------- + +def stn(s, length): + """Convert a python string to a null-terminated string buffer. + """ + return s[:length] + (length - len(s)) * NUL + +def nts(s): + """Convert a null-terminated string field to a python string. + """ + # Use the string up to the first null char. + p = s.find("\0") + if p == -1: + return s + return s[:p] + +def nti(s): + """Convert a number field to a python number. + """ + # There are two possible encodings for a number field, see + # itn() below. + if s[0] != chr(0200): + try: + n = int(nts(s) or "0", 8) + except ValueError: + raise InvalidHeaderError("invalid header") + else: + n = 0L + for i in xrange(len(s) - 1): + n <<= 8 + n += ord(s[i + 1]) + return n + +def itn(n, digits=8, format=DEFAULT_FORMAT): + """Convert a python number to a number field. + """ + # POSIX 1003.1-1988 requires numbers to be encoded as a string of + # octal digits followed by a null-byte, this allows values up to + # (8**(digits-1))-1. GNU tar allows storing numbers greater than + # that if necessary. A leading 0200 byte indicates this particular + # encoding, the following digits-1 bytes are a big-endian + # representation. This allows values up to (256**(digits-1))-1. + if 0 <= n < 8 ** (digits - 1): + s = "%0*o" % (digits - 1, n) + NUL + else: + if format != GNU_FORMAT or n >= 256 ** (digits - 1): + raise ValueError("overflow in number field") + + if n < 0: + # XXX We mimic GNU tar's behaviour with negative numbers, + # this could raise OverflowError. + n = struct.unpack("L", struct.pack("l", n))[0] + + s = "" + for i in xrange(digits - 1): + s = chr(n & 0377) + s + n >>= 8 + s = chr(0200) + s + return s + +def uts(s, encoding, errors): + """Convert a unicode object to a string. + """ + if errors == "utf-8": + # An extra error handler similar to the -o invalid=UTF-8 option + # in POSIX.1-2001. Replace untranslatable characters with their + # UTF-8 representation. + try: + return s.encode(encoding, "strict") + except UnicodeEncodeError: + x = [] + for c in s: + try: + x.append(c.encode(encoding, "strict")) + except UnicodeEncodeError: + x.append(c.encode("utf8")) + return "".join(x) + else: + return s.encode(encoding, errors) + +def calc_chksums(buf): + """Calculate the checksum for a member's header by summing up all + characters except for the chksum field which is treated as if + it was filled with spaces. According to the GNU tar sources, + some tars (Sun and NeXT) calculate chksum with signed char, + which will be different if there are chars in the buffer with + the high bit set. So we calculate two checksums, unsigned and + signed. + """ + unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512])) + signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512])) + return unsigned_chksum, signed_chksum + +def copyfileobj(src, dst, length=None): + """Copy length bytes from fileobj src to fileobj dst. + If length is None, copy the entire content. + """ + if length == 0: + return + if length is None: + shutil.copyfileobj(src, dst) + return + + BUFSIZE = 16 * 1024 + blocks, remainder = divmod(length, BUFSIZE) + for b in xrange(blocks): + buf = src.read(BUFSIZE) + if len(buf) < BUFSIZE: + raise IOError("end of file reached") + dst.write(buf) + + if remainder != 0: + buf = src.read(remainder) + if len(buf) < remainder: + raise IOError("end of file reached") + dst.write(buf) + return + +filemode_table = ( + ((S_IFLNK, "l"), + (S_IFREG, "-"), + (S_IFBLK, "b"), + (S_IFDIR, "d"), + (S_IFCHR, "c"), + (S_IFIFO, "p")), + + ((TUREAD, "r"),), + ((TUWRITE, "w"),), + ((TUEXEC|TSUID, "s"), + (TSUID, "S"), + (TUEXEC, "x")), + + ((TGREAD, "r"),), + ((TGWRITE, "w"),), + ((TGEXEC|TSGID, "s"), + (TSGID, "S"), + (TGEXEC, "x")), + + ((TOREAD, "r"),), + ((TOWRITE, "w"),), + ((TOEXEC|TSVTX, "t"), + (TSVTX, "T"), + (TOEXEC, "x")) +) + +def filemode(mode): + """Convert a file's mode to a string of the form + -rwxrwxrwx. + Used by TarFile.list() + """ + perm = [] + for table in filemode_table: + for bit, char in table: + if mode & bit == bit: + perm.append(char) + break + else: + perm.append("-") + return "".join(perm) + +class TarError(Exception): + """Base exception.""" + pass +class ExtractError(TarError): + """General exception for extract errors.""" + pass +class ReadError(TarError): + """Exception for unreadble tar archives.""" + pass +class CompressionError(TarError): + """Exception for unavailable compression methods.""" + pass +class StreamError(TarError): + """Exception for unsupported operations on stream-like TarFiles.""" + pass +class HeaderError(TarError): + """Base exception for header errors.""" + pass +class EmptyHeaderError(HeaderError): + """Exception for empty headers.""" + pass +class TruncatedHeaderError(HeaderError): + """Exception for truncated headers.""" + pass +class EOFHeaderError(HeaderError): + """Exception for end of file headers.""" + pass +class InvalidHeaderError(HeaderError): + """Exception for invalid headers.""" + pass +class SubsequentHeaderError(HeaderError): + """Exception for missing and invalid extended headers.""" + pass + +#--------------------------- +# internal stream interface +#--------------------------- +class _LowLevelFile: + """Low-level file object. Supports reading and writing. + It is used instead of a regular file object for streaming + access. + """ + + def __init__(self, name, mode): + mode = { + "r": os.O_RDONLY, + "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + }[mode] + if hasattr(os, "O_BINARY"): + mode |= os.O_BINARY + self.fd = os.open(name, mode, 0666) + + def close(self): + os.close(self.fd) + + def read(self, size): + return os.read(self.fd, size) + + def write(self, s): + os.write(self.fd, s) + +class _Stream: + """Class that serves as an adapter between TarFile and + a stream-like object. The stream-like object only + needs to have a read() or write() method and is accessed + blockwise. Use of gzip or bzip2 compression is possible. + A stream-like object could be for example: sys.stdin, + sys.stdout, a socket, a tape device etc. + + _Stream is intended to be used only internally. + """ + + def __init__(self, name, mode, comptype, fileobj, bufsize): + """Construct a _Stream object. + """ + self._extfileobj = True + if fileobj is None: + fileobj = _LowLevelFile(name, mode) + self._extfileobj = False + + if comptype == '*': + # Enable transparent compression detection for the + # stream interface + fileobj = _StreamProxy(fileobj) + comptype = fileobj.getcomptype() + + self.name = name or "" + self.mode = mode + self.comptype = comptype + self.fileobj = fileobj + self.bufsize = bufsize + self.buf = "" + self.pos = 0L + self.closed = False + + if comptype == "gz": + try: + import zlib + except ImportError: + raise CompressionError("zlib module is not available") + self.zlib = zlib + self.crc = zlib.crc32("") & 0xffffffffL + if mode == "r": + self._init_read_gz() + else: + self._init_write_gz() + + if comptype == "bz2": + try: + import bz2 + except ImportError: + raise CompressionError("bz2 module is not available") + if mode == "r": + self.dbuf = "" + self.cmp = bz2.BZ2Decompressor() + else: + self.cmp = bz2.BZ2Compressor() + + def __del__(self): + if hasattr(self, "closed") and not self.closed: + self.close() + + def _init_write_gz(self): + """Initialize for writing with gzip compression. + """ + self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED, + -self.zlib.MAX_WBITS, + self.zlib.DEF_MEM_LEVEL, + 0) + timestamp = struct.pack(" self.bufsize: + self.fileobj.write(self.buf[:self.bufsize]) + self.buf = self.buf[self.bufsize:] + + def close(self): + """Close the _Stream object. No operation should be + done on it afterwards. + """ + if self.closed: + return + + if self.mode == "w" and self.comptype != "tar": + self.buf += self.cmp.flush() + + if self.mode == "w" and self.buf: + self.fileobj.write(self.buf) + self.buf = "" + if self.comptype == "gz": + # The native zlib crc is an unsigned 32-bit integer, but + # the Python wrapper implicitly casts that to a signed C + # long. So, on a 32-bit box self.crc may "look negative", + # while the same crc on a 64-bit box may "look positive". + # To avoid irksome warnings from the `struct` module, force + # it to look positive on all boxes. + self.fileobj.write(struct.pack("= 0: + blocks, remainder = divmod(pos - self.pos, self.bufsize) + for i in xrange(blocks): + self.read(self.bufsize) + self.read(remainder) + else: + raise StreamError("seeking backwards is not allowed") + return self.pos + + def read(self, size=None): + """Return the next size number of bytes from the stream. + If size is not defined, return all bytes of the stream + up to EOF. + """ + if size is None: + t = [] + while True: + buf = self._read(self.bufsize) + if not buf: + break + t.append(buf) + buf = "".join(t) + else: + buf = self._read(size) + self.pos += len(buf) + return buf + + def _read(self, size): + """Return size bytes from the stream. + """ + if self.comptype == "tar": + return self.__read(size) + + c = len(self.dbuf) + t = [self.dbuf] + while c < size: + buf = self.__read(self.bufsize) + if not buf: + break + try: + buf = self.cmp.decompress(buf) + except IOError: + raise ReadError("invalid compressed data") + t.append(buf) + c += len(buf) + t = "".join(t) + self.dbuf = t[size:] + return t[:size] + + def __read(self, size): + """Return size bytes from stream. If internal buffer is empty, + read another block from the stream. + """ + c = len(self.buf) + t = [self.buf] + while c < size: + buf = self.fileobj.read(self.bufsize) + if not buf: + break + t.append(buf) + c += len(buf) + t = "".join(t) + self.buf = t[size:] + return t[:size] +# class _Stream + +class _StreamProxy(object): + """Small proxy class that enables transparent compression + detection for the Stream interface (mode 'r|*'). + """ + + def __init__(self, fileobj): + self.fileobj = fileobj + self.buf = self.fileobj.read(BLOCKSIZE) + + def read(self, size): + self.read = self.fileobj.read + return self.buf + + def getcomptype(self): + if self.buf.startswith("\037\213\010"): + return "gz" + if self.buf.startswith("BZh91"): + return "bz2" + return "tar" + + def close(self): + self.fileobj.close() +# class StreamProxy + +class _BZ2Proxy(object): + """Small proxy class that enables external file object + support for "r:bz2" and "w:bz2" modes. This is actually + a workaround for a limitation in bz2 module's BZ2File + class which (unlike gzip.GzipFile) has no support for + a file object argument. + """ + + blocksize = 16 * 1024 + + def __init__(self, fileobj, mode): + self.fileobj = fileobj + self.mode = mode + self.name = getattr(self.fileobj, "name", None) + self.init() + + def init(self): + import bz2 + self.pos = 0 + if self.mode == "r": + self.bz2obj = bz2.BZ2Decompressor() + self.fileobj.seek(0) + self.buf = "" + else: + self.bz2obj = bz2.BZ2Compressor() + + def read(self, size): + b = [self.buf] + x = len(self.buf) + while x < size: + raw = self.fileobj.read(self.blocksize) + if not raw: + break + data = self.bz2obj.decompress(raw) + b.append(data) + x += len(data) + self.buf = "".join(b) + + buf = self.buf[:size] + self.buf = self.buf[size:] + self.pos += len(buf) + return buf + + def seek(self, pos): + if pos < self.pos: + self.init() + self.read(pos - self.pos) + + def tell(self): + return self.pos + + def write(self, data): + self.pos += len(data) + raw = self.bz2obj.compress(data) + self.fileobj.write(raw) + + def close(self): + if self.mode == "w": + raw = self.bz2obj.flush() + self.fileobj.write(raw) +# class _BZ2Proxy + +#------------------------ +# Extraction file object +#------------------------ +class _FileInFile(object): + """A thin wrapper around an existing file object that + provides a part of its data as an individual file + object. + """ + + def __init__(self, fileobj, offset, size, sparse=None): + self.fileobj = fileobj + self.offset = offset + self.size = size + self.sparse = sparse + self.position = 0 + + def tell(self): + """Return the current file position. + """ + return self.position + + def seek(self, position): + """Seek to a position in the file. + """ + self.position = position + + def read(self, size=None): + """Read data from the file. + """ + if size is None: + size = self.size - self.position + else: + size = min(size, self.size - self.position) + + if self.sparse is None: + return self.readnormal(size) + else: + return self.readsparse(size) + + def readnormal(self, size): + """Read operation for regular files. + """ + self.fileobj.seek(self.offset + self.position) + self.position += size + return self.fileobj.read(size) + + def readsparse(self, size): + """Read operation for sparse files. + """ + data = [] + while size > 0: + buf = self.readsparsesection(size) + if not buf: + break + size -= len(buf) + data.append(buf) + return "".join(data) + + def readsparsesection(self, size): + """Read a single section of a sparse file. + """ + section = self.sparse.find(self.position) + + if section is None: + return "" + + size = min(size, section.offset + section.size - self.position) + + if isinstance(section, _data): + realpos = section.realpos + self.position - section.offset + self.fileobj.seek(self.offset + realpos) + self.position += size + return self.fileobj.read(size) + else: + self.position += size + return NUL * size +#class _FileInFile + + +class ExFileObject(object): + """File-like object for reading an archive member. + Is returned by TarFile.extractfile(). + """ + blocksize = 1024 + + def __init__(self, tarfile, tarinfo): + self.fileobj = _FileInFile(tarfile.fileobj, + tarinfo.offset_data, + tarinfo.size, + getattr(tarinfo, "sparse", None)) + self.name = tarinfo.name + self.mode = "r" + self.closed = False + self.size = tarinfo.size + + self.position = 0 + self.buffer = "" + + def read(self, size=None): + """Read at most size bytes from the file. If size is not + present or None, read all data until EOF is reached. + """ + if self.closed: + raise ValueError("I/O operation on closed file") + + buf = "" + if self.buffer: + if size is None: + buf = self.buffer + self.buffer = "" + else: + buf = self.buffer[:size] + self.buffer = self.buffer[size:] + + if size is None: + buf += self.fileobj.read() + else: + buf += self.fileobj.read(size - len(buf)) + + self.position += len(buf) + return buf + + def readline(self, size=-1): + """Read one entire line from the file. If size is present + and non-negative, return a string with at most that + size, which may be an incomplete line. + """ + if self.closed: + raise ValueError("I/O operation on closed file") + + if "\n" in self.buffer: + pos = self.buffer.find("\n") + 1 + else: + buffers = [self.buffer] + while True: + buf = self.fileobj.read(self.blocksize) + buffers.append(buf) + if not buf or "\n" in buf: + self.buffer = "".join(buffers) + pos = self.buffer.find("\n") + 1 + if pos == 0: + # no newline found. + pos = len(self.buffer) + break + + if size != -1: + pos = min(size, pos) + + buf = self.buffer[:pos] + self.buffer = self.buffer[pos:] + self.position += len(buf) + return buf + + def readlines(self): + """Return a list with all remaining lines. + """ + result = [] + while True: + line = self.readline() + if not line: break + result.append(line) + return result + + def tell(self): + """Return the current file position. + """ + if self.closed: + raise ValueError("I/O operation on closed file") + + return self.position + + def seek(self, pos, whence=os.SEEK_SET): + """Seek to a position in the file. + """ + if self.closed: + raise ValueError("I/O operation on closed file") + + if whence == os.SEEK_SET: + self.position = min(max(pos, 0), self.size) + elif whence == os.SEEK_CUR: + if pos < 0: + self.position = max(self.position + pos, 0) + else: + self.position = min(self.position + pos, self.size) + elif whence == os.SEEK_END: + self.position = max(min(self.size + pos, self.size), 0) + else: + raise ValueError("Invalid argument") + + self.buffer = "" + self.fileobj.seek(self.position) + + def close(self): + """Close the file object. + """ + self.closed = True + + def __iter__(self): + """Get an iterator over the file's lines. + """ + while True: + line = self.readline() + if not line: + break + yield line +#class ExFileObject + +#------------------ +# Exported Classes +#------------------ +class TarInfo(object): + """Informational class which holds the details about an + archive member given by a tar header block. + TarInfo objects are returned by TarFile.getmember(), + TarFile.getmembers() and TarFile.gettarinfo() and are + usually created internally. + """ + + def __init__(self, name=""): + """Construct a TarInfo object. name is the optional name + of the member. + """ + self.name = name # member name + self.mode = 0644 # file permissions + self.uid = 0 # user id + self.gid = 0 # group id + self.size = 0 # file size + self.mtime = 0 # modification time + self.chksum = 0 # header checksum + self.type = REGTYPE # member type + self.linkname = "" # link name + self.uname = "" # user name + self.gname = "" # group name + self.devmajor = 0 # device major number + self.devminor = 0 # device minor number + + self.offset = 0 # the tar header starts here + self.offset_data = 0 # the file's data starts here + + self.pax_headers = {} # pax header information + + # In pax headers the "name" and "linkname" field are called + # "path" and "linkpath". + def _getpath(self): + return self.name + def _setpath(self, name): + self.name = name + path = property(_getpath, _setpath) + + def _getlinkpath(self): + return self.linkname + def _setlinkpath(self, linkname): + self.linkname = linkname + linkpath = property(_getlinkpath, _setlinkpath) + + def __repr__(self): + return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self)) + + def get_info(self, encoding, errors): + """Return the TarInfo's attributes as a dictionary. + """ + info = { + "name": self.name, + "mode": self.mode & 07777, + "uid": self.uid, + "gid": self.gid, + "size": self.size, + "mtime": self.mtime, + "chksum": self.chksum, + "type": self.type, + "linkname": self.linkname, + "uname": self.uname, + "gname": self.gname, + "devmajor": self.devmajor, + "devminor": self.devminor + } + + if info["type"] == DIRTYPE and not info["name"].endswith("/"): + info["name"] += "/" + + for key in ("name", "linkname", "uname", "gname"): + if type(info[key]) is unicode: + info[key] = info[key].encode(encoding, errors) + + return info + + def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="strict"): + """Return a tar header as a string of 512 byte blocks. + """ + info = self.get_info(encoding, errors) + + if format == USTAR_FORMAT: + return self.create_ustar_header(info) + elif format == GNU_FORMAT: + return self.create_gnu_header(info) + elif format == PAX_FORMAT: + return self.create_pax_header(info, encoding, errors) + else: + raise ValueError("invalid format") + + def create_ustar_header(self, info): + """Return the object as a ustar header block. + """ + info["magic"] = POSIX_MAGIC + + if len(info["linkname"]) > LENGTH_LINK: + raise ValueError("linkname is too long") + + if len(info["name"]) > LENGTH_NAME: + info["prefix"], info["name"] = self._posix_split_name(info["name"]) + + return self._create_header(info, USTAR_FORMAT) + + def create_gnu_header(self, info): + """Return the object as a GNU header block sequence. + """ + info["magic"] = GNU_MAGIC + + buf = "" + if len(info["linkname"]) > LENGTH_LINK: + buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK) + + if len(info["name"]) > LENGTH_NAME: + buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME) + + return buf + self._create_header(info, GNU_FORMAT) + + def create_pax_header(self, info, encoding, errors): + """Return the object as a ustar header block. If it cannot be + represented this way, prepend a pax extended header sequence + with supplement information. + """ + info["magic"] = POSIX_MAGIC + pax_headers = self.pax_headers.copy() + + # Test string fields for values that exceed the field length or cannot + # be represented in ASCII encoding. + for name, hname, length in ( + ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK), + ("uname", "uname", 32), ("gname", "gname", 32)): + + if hname in pax_headers: + # The pax header has priority. + continue + + val = info[name].decode(encoding, errors) + + # Try to encode the string as ASCII. + try: + val.encode("ascii") + except UnicodeEncodeError: + pax_headers[hname] = val + continue + + if len(info[name]) > length: + pax_headers[hname] = val + + # Test number fields for values that exceed the field limit or values + # that like to be stored as float. + for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)): + if name in pax_headers: + # The pax header has priority. Avoid overflow. + info[name] = 0 + continue + + val = info[name] + if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float): + pax_headers[name] = unicode(val) + info[name] = 0 + + # Create a pax extended header if necessary. + if pax_headers: + buf = self._create_pax_generic_header(pax_headers) + else: + buf = "" + + return buf + self._create_header(info, USTAR_FORMAT) + + @classmethod + def create_pax_global_header(cls, pax_headers): + """Return the object as a pax global header block sequence. + """ + return cls._create_pax_generic_header(pax_headers, type=XGLTYPE) + + def _posix_split_name(self, name): + """Split a name longer than 100 chars into a prefix + and a name part. + """ + prefix = name[:LENGTH_PREFIX + 1] + while prefix and prefix[-1] != "/": + prefix = prefix[:-1] + + name = name[len(prefix):] + prefix = prefix[:-1] + + if not prefix or len(name) > LENGTH_NAME: + raise ValueError("name is too long") + return prefix, name + + @staticmethod + def _create_header(info, format): + """Return a header block. info is a dictionary with file + information, format must be one of the *_FORMAT constants. + """ + parts = [ + stn(info.get("name", ""), 100), + itn(info.get("mode", 0) & 07777, 8, format), + itn(info.get("uid", 0), 8, format), + itn(info.get("gid", 0), 8, format), + itn(info.get("size", 0), 12, format), + itn(info.get("mtime", 0), 12, format), + " ", # checksum field + info.get("type", REGTYPE), + stn(info.get("linkname", ""), 100), + stn(info.get("magic", POSIX_MAGIC), 8), + stn(info.get("uname", ""), 32), + stn(info.get("gname", ""), 32), + itn(info.get("devmajor", 0), 8, format), + itn(info.get("devminor", 0), 8, format), + stn(info.get("prefix", ""), 155) + ] + + buf = struct.pack("%ds" % BLOCKSIZE, "".join(parts)) + chksum = calc_chksums(buf[-BLOCKSIZE:])[0] + buf = buf[:-364] + "%06o\0" % chksum + buf[-357:] + return buf + + @staticmethod + def _create_payload(payload): + """Return the string payload filled with zero bytes + up to the next 512 byte border. + """ + blocks, remainder = divmod(len(payload), BLOCKSIZE) + if remainder > 0: + payload += (BLOCKSIZE - remainder) * NUL + return payload + + @classmethod + def _create_gnu_long_header(cls, name, type): + """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence + for name. + """ + name += NUL + + info = {} + info["name"] = "././@LongLink" + info["type"] = type + info["size"] = len(name) + info["magic"] = GNU_MAGIC + + # create extended header + name blocks. + return cls._create_header(info, USTAR_FORMAT) + \ + cls._create_payload(name) + + @classmethod + def _create_pax_generic_header(cls, pax_headers, type=XHDTYPE): + """Return a POSIX.1-2001 extended or global header sequence + that contains a list of keyword, value pairs. The values + must be unicode objects. + """ + records = [] + for keyword, value in pax_headers.iteritems(): + keyword = keyword.encode("utf8") + value = value.encode("utf8") + l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n' + n = p = 0 + while True: + n = l + len(str(p)) + if n == p: + break + p = n + records.append("%d %s=%s\n" % (p, keyword, value)) + records = "".join(records) + + # We use a hardcoded "././@PaxHeader" name like star does + # instead of the one that POSIX recommends. + info = {} + info["name"] = "././@PaxHeader" + info["type"] = type + info["size"] = len(records) + info["magic"] = POSIX_MAGIC + + # Create pax header + record blocks. + return cls._create_header(info, USTAR_FORMAT) + \ + cls._create_payload(records) + + @classmethod + def frombuf(cls, buf): + """Construct a TarInfo object from a 512 byte string buffer. + """ + if len(buf) == 0: + raise EmptyHeaderError("empty header") + if len(buf) != BLOCKSIZE: + raise TruncatedHeaderError("truncated header") + if buf.count(NUL) == BLOCKSIZE: + raise EOFHeaderError("end of file header") + + chksum = nti(buf[148:156]) + if chksum not in calc_chksums(buf): + raise InvalidHeaderError("bad checksum") + + obj = cls() + obj.buf = buf + obj.name = nts(buf[0:100]) + obj.mode = nti(buf[100:108]) + obj.uid = nti(buf[108:116]) + obj.gid = nti(buf[116:124]) + obj.size = nti(buf[124:136]) + obj.mtime = nti(buf[136:148]) + obj.chksum = chksum + obj.type = buf[156:157] + obj.linkname = nts(buf[157:257]) + obj.uname = nts(buf[265:297]) + obj.gname = nts(buf[297:329]) + obj.devmajor = nti(buf[329:337]) + obj.devminor = nti(buf[337:345]) + prefix = nts(buf[345:500]) + + # Old V7 tar format represents a directory as a regular + # file with a trailing slash. + if obj.type == AREGTYPE and obj.name.endswith("/"): + obj.type = DIRTYPE + + # Remove redundant slashes from directories. + if obj.isdir(): + obj.name = obj.name.rstrip("/") + + # Reconstruct a ustar longname. + if prefix and obj.type not in GNU_TYPES: + obj.name = prefix + "/" + obj.name + return obj + + @classmethod + def fromtarfile(cls, tarfile): + """Return the next TarInfo object from TarFile object + tarfile. + """ + buf = tarfile.fileobj.read(BLOCKSIZE) + obj = cls.frombuf(buf) + obj.offset = tarfile.fileobj.tell() - BLOCKSIZE + return obj._proc_member(tarfile) + + #-------------------------------------------------------------------------- + # The following are methods that are called depending on the type of a + # member. The entry point is _proc_member() which can be overridden in a + # subclass to add custom _proc_*() methods. A _proc_*() method MUST + # implement the following + # operations: + # 1. Set self.offset_data to the position where the data blocks begin, + # if there is data that follows. + # 2. Set tarfile.offset to the position where the next member's header will + # begin. + # 3. Return self or another valid TarInfo object. + def _proc_member(self, tarfile): + """Choose the right processing method depending on + the type and call it. + """ + if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK): + return self._proc_gnulong(tarfile) + elif self.type == GNUTYPE_SPARSE: + return self._proc_sparse(tarfile) + elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE): + return self._proc_pax(tarfile) + else: + return self._proc_builtin(tarfile) + + def _proc_builtin(self, tarfile): + """Process a builtin type or an unknown type which + will be treated as a regular file. + """ + self.offset_data = tarfile.fileobj.tell() + offset = self.offset_data + if self.isreg() or self.type not in SUPPORTED_TYPES: + # Skip the following data blocks. + offset += self._block(self.size) + tarfile.offset = offset + + # Patch the TarInfo object with saved global + # header information. + self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors) + + return self + + def _proc_gnulong(self, tarfile): + """Process the blocks that hold a GNU longname + or longlink member. + """ + buf = tarfile.fileobj.read(self._block(self.size)) + + # Fetch the next header and process it. + try: + next = self.fromtarfile(tarfile) + except HeaderError: + raise SubsequentHeaderError("missing or bad subsequent header") + + # Patch the TarInfo object from the next header with + # the longname information. + next.offset = self.offset + if self.type == GNUTYPE_LONGNAME: + next.name = nts(buf) + elif self.type == GNUTYPE_LONGLINK: + next.linkname = nts(buf) + + return next + + def _proc_sparse(self, tarfile): + """Process a GNU sparse header plus extra headers. + """ + buf = self.buf + sp = _ringbuffer() + pos = 386 + lastpos = 0L + realpos = 0L + # There are 4 possible sparse structs in the + # first header. + for i in xrange(4): + try: + offset = nti(buf[pos:pos + 12]) + numbytes = nti(buf[pos + 12:pos + 24]) + except ValueError: + break + if offset > lastpos: + sp.append(_hole(lastpos, offset - lastpos)) + sp.append(_data(offset, numbytes, realpos)) + realpos += numbytes + lastpos = offset + numbytes + pos += 24 + + isextended = ord(buf[482]) + origsize = nti(buf[483:495]) + + # If the isextended flag is given, + # there are extra headers to process. + while isextended == 1: + buf = tarfile.fileobj.read(BLOCKSIZE) + pos = 0 + for i in xrange(21): + try: + offset = nti(buf[pos:pos + 12]) + numbytes = nti(buf[pos + 12:pos + 24]) + except ValueError: + break + if offset > lastpos: + sp.append(_hole(lastpos, offset - lastpos)) + sp.append(_data(offset, numbytes, realpos)) + realpos += numbytes + lastpos = offset + numbytes + pos += 24 + isextended = ord(buf[504]) + + if lastpos < origsize: + sp.append(_hole(lastpos, origsize - lastpos)) + + self.sparse = sp + + self.offset_data = tarfile.fileobj.tell() + tarfile.offset = self.offset_data + self._block(self.size) + self.size = origsize + + return self + + def _proc_pax(self, tarfile): + """Process an extended or global header as described in + POSIX.1-2001. + """ + # Read the header information. + buf = tarfile.fileobj.read(self._block(self.size)) + + # A pax header stores supplemental information for either + # the following file (extended) or all following files + # (global). + if self.type == XGLTYPE: + pax_headers = tarfile.pax_headers + else: + pax_headers = tarfile.pax_headers.copy() + + # Parse pax header information. A record looks like that: + # "%d %s=%s\n" % (length, keyword, value). length is the size + # of the complete record including the length field itself and + # the newline. keyword and value are both UTF-8 encoded strings. + regex = re.compile(r"(\d+) ([^=]+)=", re.U) + pos = 0 + while True: + match = regex.match(buf, pos) + if not match: + break + + length, keyword = match.groups() + length = int(length) + value = buf[match.end(2) + 1:match.start(1) + length - 1] + + keyword = keyword.decode("utf8") + value = value.decode("utf8") + + pax_headers[keyword] = value + pos += length + + # Fetch the next header. + try: + next = self.fromtarfile(tarfile) + except HeaderError: + raise SubsequentHeaderError("missing or bad subsequent header") + + if self.type in (XHDTYPE, SOLARIS_XHDTYPE): + # Patch the TarInfo object with the extended header info. + next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) + next.offset = self.offset + + if "size" in pax_headers: + # If the extended header replaces the size field, + # we need to recalculate the offset where the next + # header starts. + offset = next.offset_data + if next.isreg() or next.type not in SUPPORTED_TYPES: + offset += next._block(next.size) + tarfile.offset = offset + + return next + + def _apply_pax_info(self, pax_headers, encoding, errors): + """Replace fields with supplemental information from a previous + pax extended or global header. + """ + for keyword, value in pax_headers.iteritems(): + if keyword not in PAX_FIELDS: + continue + + if keyword == "path": + value = value.rstrip("/") + + if keyword in PAX_NUMBER_FIELDS: + try: + value = PAX_NUMBER_FIELDS[keyword](value) + except ValueError: + value = 0 + else: + value = uts(value, encoding, errors) + + setattr(self, keyword, value) + + self.pax_headers = pax_headers.copy() + + def _block(self, count): + """Round up a byte count by BLOCKSIZE and return it, + e.g. _block(834) => 1024. + """ + blocks, remainder = divmod(count, BLOCKSIZE) + if remainder: + blocks += 1 + return blocks * BLOCKSIZE + + def isreg(self): + return self.type in REGULAR_TYPES + def isfile(self): + return self.isreg() + def isdir(self): + return self.type == DIRTYPE + def issym(self): + return self.type == SYMTYPE + def islnk(self): + return self.type == LNKTYPE + def ischr(self): + return self.type == CHRTYPE + def isblk(self): + return self.type == BLKTYPE + def isfifo(self): + return self.type == FIFOTYPE + def issparse(self): + return self.type == GNUTYPE_SPARSE + def isdev(self): + return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE) +# class TarInfo + +class TarFile(object): + """The TarFile Class provides an interface to tar archives. + """ + + debug = 0 # May be set from 0 (no msgs) to 3 (all msgs) + + dereference = False # If true, add content of linked file to the + # tar file, else the link. + + ignore_zeros = False # If true, skips empty or invalid blocks and + # continues processing. + + errorlevel = 1 # If 0, fatal errors only appear in debug + # messages (if debug >= 0). If > 0, errors + # are passed to the caller as exceptions. + + format = DEFAULT_FORMAT # The format to use when creating an archive. + + encoding = ENCODING # Encoding for 8-bit character strings. + + errors = None # Error handler for unicode conversion. + + tarinfo = TarInfo # The default TarInfo class to use. + + fileobject = ExFileObject # The default ExFileObject class to use. + + def __init__(self, name=None, mode="r", fileobj=None, format=None, + tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, + errors=None, pax_headers=None, debug=None, errorlevel=None): + """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to + read from an existing archive, 'a' to append data to an existing + file or 'w' to create a new file overwriting an existing one. `mode' + defaults to 'r'. + If `fileobj' is given, it is used for reading or writing data. If it + can be determined, `mode' is overridden by `fileobj's mode. + `fileobj' is not closed, when TarFile is closed. + """ + if len(mode) > 1 or mode not in "raw": + raise ValueError("mode must be 'r', 'a' or 'w'") + self.mode = mode + self._mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode] + + if not fileobj: + if self.mode == "a" and not os.path.exists(name): + # Create nonexistent files in append mode. + self.mode = "w" + self._mode = "wb" + fileobj = bltn_open(name, self._mode) + self._extfileobj = False + else: + if name is None and hasattr(fileobj, "name"): + name = fileobj.name + if hasattr(fileobj, "mode"): + self._mode = fileobj.mode + self._extfileobj = True + self.name = os.path.abspath(name) if name else None + self.fileobj = fileobj + + # Init attributes. + if format is not None: + self.format = format + if tarinfo is not None: + self.tarinfo = tarinfo + if dereference is not None: + self.dereference = dereference + if ignore_zeros is not None: + self.ignore_zeros = ignore_zeros + if encoding is not None: + self.encoding = encoding + + if errors is not None: + self.errors = errors + elif mode == "r": + self.errors = "utf-8" + else: + self.errors = "strict" + + if pax_headers is not None and self.format == PAX_FORMAT: + self.pax_headers = pax_headers + else: + self.pax_headers = {} + + if debug is not None: + self.debug = debug + if errorlevel is not None: + self.errorlevel = errorlevel + + # Init datastructures. + self.closed = False + self.members = [] # list of members as TarInfo objects + self._loaded = False # flag if all members have been read + self.offset = self.fileobj.tell() + # current position in the archive file + self.inodes = {} # dictionary caching the inodes of + # archive members already added + + try: + if self.mode == "r": + self.firstmember = None + self.firstmember = self.next() + + if self.mode == "a": + # Move to the end of the archive, + # before the first empty block. + while True: + self.fileobj.seek(self.offset) + try: + tarinfo = self.tarinfo.fromtarfile(self) + self.members.append(tarinfo) + except EOFHeaderError: + self.fileobj.seek(self.offset) + break + except HeaderError, e: + raise ReadError(str(e)) + + if self.mode in "aw": + self._loaded = True + + if self.pax_headers: + buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy()) + self.fileobj.write(buf) + self.offset += len(buf) + except: + if not self._extfileobj: + self.fileobj.close() + self.closed = True + raise + + def _getposix(self): + return self.format == USTAR_FORMAT + def _setposix(self, value): + import warnings + warnings.warn("use the format attribute instead", DeprecationWarning, + 2) + if value: + self.format = USTAR_FORMAT + else: + self.format = GNU_FORMAT + posix = property(_getposix, _setposix) + + #-------------------------------------------------------------------------- + # Below are the classmethods which act as alternate constructors to the + # TarFile class. The open() method is the only one that is needed for + # public use; it is the "super"-constructor and is able to select an + # adequate "sub"-constructor for a particular compression using the mapping + # from OPEN_METH. + # + # This concept allows one to subclass TarFile without losing the comfort of + # the super-constructor. A sub-constructor is registered and made available + # by adding it to the mapping in OPEN_METH. + + @classmethod + def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): + """Open a tar archive for reading, writing or appending. Return + an appropriate TarFile class. + + mode: + 'r' or 'r:*' open for reading with transparent compression + 'r:' open for reading exclusively uncompressed + 'r:gz' open for reading with gzip compression + 'r:bz2' open for reading with bzip2 compression + 'a' or 'a:' open for appending, creating the file if necessary + 'w' or 'w:' open for writing without compression + 'w:gz' open for writing with gzip compression + 'w:bz2' open for writing with bzip2 compression + + 'r|*' open a stream of tar blocks with transparent compression + 'r|' open an uncompressed stream of tar blocks for reading + 'r|gz' open a gzip compressed stream of tar blocks + 'r|bz2' open a bzip2 compressed stream of tar blocks + 'w|' open an uncompressed stream for writing + 'w|gz' open a gzip compressed stream for writing + 'w|bz2' open a bzip2 compressed stream for writing + """ + + if not name and not fileobj: + raise ValueError("nothing to open") + + if mode in ("r", "r:*"): + # Find out which *open() is appropriate for opening the file. + for comptype in cls.OPEN_METH: + func = getattr(cls, cls.OPEN_METH[comptype]) + if fileobj is not None: + saved_pos = fileobj.tell() + try: + return func(name, "r", fileobj, **kwargs) + except (ReadError, CompressionError), e: + if fileobj is not None: + fileobj.seek(saved_pos) + continue + raise ReadError("file could not be opened successfully") + + elif ":" in mode: + filemode, comptype = mode.split(":", 1) + filemode = filemode or "r" + comptype = comptype or "tar" + + # Select the *open() function according to + # given compression. + if comptype in cls.OPEN_METH: + func = getattr(cls, cls.OPEN_METH[comptype]) + else: + raise CompressionError("unknown compression type %r" % comptype) + return func(name, filemode, fileobj, **kwargs) + + elif "|" in mode: + filemode, comptype = mode.split("|", 1) + filemode = filemode or "r" + comptype = comptype or "tar" + + if filemode not in "rw": + raise ValueError("mode must be 'r' or 'w'") + + t = cls(name, filemode, + _Stream(name, filemode, comptype, fileobj, bufsize), + **kwargs) + t._extfileobj = False + return t + + elif mode in "aw": + return cls.taropen(name, mode, fileobj, **kwargs) + + raise ValueError("undiscernible mode") + + @classmethod + def taropen(cls, name, mode="r", fileobj=None, **kwargs): + """Open uncompressed tar archive name for reading or writing. + """ + if len(mode) > 1 or mode not in "raw": + raise ValueError("mode must be 'r', 'a' or 'w'") + return cls(name, mode, fileobj, **kwargs) + + @classmethod + def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): + """Open gzip compressed tar archive name for reading or writing. + Appending is not allowed. + """ + if len(mode) > 1 or mode not in "rw": + raise ValueError("mode must be 'r' or 'w'") + + try: + import gzip + gzip.GzipFile + except (ImportError, AttributeError): + raise CompressionError("gzip module is not available") + + if fileobj is None: + fileobj = bltn_open(name, mode + "b") + + try: + t = cls.taropen(name, mode, + gzip.GzipFile(name, mode, compresslevel, fileobj), + **kwargs) + except IOError: + raise ReadError("not a gzip file") + t._extfileobj = False + return t + + @classmethod + def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): + """Open bzip2 compressed tar archive name for reading or writing. + Appending is not allowed. + """ + if len(mode) > 1 or mode not in "rw": + raise ValueError("mode must be 'r' or 'w'.") + + try: + import bz2 + except ImportError: + raise CompressionError("bz2 module is not available") + + if fileobj is not None: + fileobj = _BZ2Proxy(fileobj, mode) + else: + fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel) + + try: + t = cls.taropen(name, mode, fileobj, **kwargs) + except (IOError, EOFError): + raise ReadError("not a bzip2 file") + t._extfileobj = False + return t + + # All *open() methods are registered here. + OPEN_METH = { + "tar": "taropen", # uncompressed tar + "gz": "gzopen", # gzip compressed tar + "bz2": "bz2open" # bzip2 compressed tar + } + + #-------------------------------------------------------------------------- + # The public methods which TarFile provides: + + def close(self): + """Close the TarFile. In write-mode, two finishing zero blocks are + appended to the archive. + """ + if self.closed: + return + + if self.mode in "aw": + self.fileobj.write(NUL * (BLOCKSIZE * 2)) + self.offset += (BLOCKSIZE * 2) + # fill up the end with zero-blocks + # (like option -b20 for tar does) + blocks, remainder = divmod(self.offset, RECORDSIZE) + if remainder > 0: + self.fileobj.write(NUL * (RECORDSIZE - remainder)) + + if not self._extfileobj: + self.fileobj.close() + self.closed = True + + def getmember(self, name): + """Return a TarInfo object for member `name'. If `name' can not be + found in the archive, KeyError is raised. If a member occurs more + than once in the archive, its last occurrence is assumed to be the + most up-to-date version. + """ + tarinfo = self._getmember(name) + if tarinfo is None: + raise KeyError("filename %r not found" % name) + return tarinfo + + def getmembers(self): + """Return the members of the archive as a list of TarInfo objects. The + list has the same order as the members in the archive. + """ + self._check() + if not self._loaded: # if we want to obtain a list of + self._load() # all members, we first have to + # scan the whole archive. + return self.members + + def getnames(self): + """Return the members of the archive as a list of their names. It has + the same order as the list returned by getmembers(). + """ + return [tarinfo.name for tarinfo in self.getmembers()] + + def gettarinfo(self, name=None, arcname=None, fileobj=None): + """Create a TarInfo object for either the file `name' or the file + object `fileobj' (using os.fstat on its file descriptor). You can + modify some of the TarInfo's attributes before you add it using + addfile(). If given, `arcname' specifies an alternative name for the + file in the archive. + """ + self._check("aw") + + # When fileobj is given, replace name by + # fileobj's real name. + if fileobj is not None: + name = fileobj.name + + # Building the name of the member in the archive. + # Backward slashes are converted to forward slashes, + # Absolute paths are turned to relative paths. + if arcname is None: + arcname = name + drv, arcname = os.path.splitdrive(arcname) + arcname = arcname.replace(os.sep, "/") + arcname = arcname.lstrip("/") + + # Now, fill the TarInfo object with + # information specific for the file. + tarinfo = self.tarinfo() + tarinfo.tarfile = self + + # Use os.stat or os.lstat, depending on platform + # and if symlinks shall be resolved. + if fileobj is None: + if hasattr(os, "lstat") and not self.dereference: + statres = os.lstat(name) + else: + statres = os.stat(name) + else: + statres = os.fstat(fileobj.fileno()) + linkname = "" + + stmd = statres.st_mode + if stat.S_ISREG(stmd): + inode = (statres.st_ino, statres.st_dev) + if not self.dereference and statres.st_nlink > 1 and \ + inode in self.inodes and arcname != self.inodes[inode]: + # Is it a hardlink to an already + # archived file? + type = LNKTYPE + linkname = self.inodes[inode] + else: + # The inode is added only if its valid. + # For win32 it is always 0. + type = REGTYPE + if inode[0]: + self.inodes[inode] = arcname + elif stat.S_ISDIR(stmd): + type = DIRTYPE + elif stat.S_ISFIFO(stmd): + type = FIFOTYPE + elif stat.S_ISLNK(stmd): + type = SYMTYPE + linkname = os.readlink(name) + elif stat.S_ISCHR(stmd): + type = CHRTYPE + elif stat.S_ISBLK(stmd): + type = BLKTYPE + else: + return None + + # Fill the TarInfo object with all + # information we can get. + tarinfo.name = arcname + tarinfo.mode = stmd + tarinfo.uid = statres.st_uid + tarinfo.gid = statres.st_gid + if type == REGTYPE: + tarinfo.size = statres.st_size + else: + tarinfo.size = 0L + tarinfo.mtime = statres.st_mtime + tarinfo.type = type + tarinfo.linkname = linkname + if pwd: + try: + tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] + except KeyError: + pass + if grp: + try: + tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] + except KeyError: + pass + + if type in (CHRTYPE, BLKTYPE): + if hasattr(os, "major") and hasattr(os, "minor"): + tarinfo.devmajor = os.major(statres.st_rdev) + tarinfo.devminor = os.minor(statres.st_rdev) + return tarinfo + + def list(self, verbose=True): + """Print a table of contents to sys.stdout. If `verbose' is False, only + the names of the members are printed. If it is True, an `ls -l'-like + output is produced. + """ + self._check() + + for tarinfo in self: + if verbose: + print filemode(tarinfo.mode), + print "%s/%s" % (tarinfo.uname or tarinfo.uid, + tarinfo.gname or tarinfo.gid), + if tarinfo.ischr() or tarinfo.isblk(): + print "%10s" % ("%d,%d" \ + % (tarinfo.devmajor, tarinfo.devminor)), + else: + print "%10d" % tarinfo.size, + print "%d-%02d-%02d %02d:%02d:%02d" \ + % time.localtime(tarinfo.mtime)[:6], + + print tarinfo.name + ("/" if tarinfo.isdir() else ""), + + if verbose: + if tarinfo.issym(): + print "->", tarinfo.linkname, + if tarinfo.islnk(): + print "link to", tarinfo.linkname, + print + + def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): + """Add the file `name' to the archive. `name' may be any type of file + (directory, fifo, symbolic link, etc.). If given, `arcname' + specifies an alternative name for the file in the archive. + Directories are added recursively by default. This can be avoided by + setting `recursive' to False. `exclude' is a function that should + return True for each filename to be excluded. `filter' is a function + that expects a TarInfo object argument and returns the changed + TarInfo object, if it returns None the TarInfo object will be + excluded from the archive. + """ + self._check("aw") + + if arcname is None: + arcname = name + + # Exclude pathnames. + if exclude is not None: + import warnings + warnings.warn("use the filter argument instead", + DeprecationWarning, 2) + if exclude(name): + self._dbg(2, "tarfile: Excluded %r" % name) + return + + # Skip if somebody tries to archive the archive... + if self.name is not None and os.path.abspath(name) == self.name: + self._dbg(2, "tarfile: Skipped %r" % name) + return + + self._dbg(1, name) + + # Create a TarInfo object from the file. + tarinfo = self.gettarinfo(name, arcname) + + if tarinfo is None: + self._dbg(1, "tarfile: Unsupported type %r" % name) + return + + # Change or exclude the TarInfo object. + if filter is not None: + tarinfo = filter(tarinfo) + if tarinfo is None: + self._dbg(2, "tarfile: Excluded %r" % name) + return + + # Append the tar header and data to the archive. + if tarinfo.isreg(): + f = bltn_open(name, "rb") + self.addfile(tarinfo, f) + f.close() + + elif tarinfo.isdir(): + self.addfile(tarinfo) + if recursive: + for f in os.listdir(name): + self.add(os.path.join(name, f), os.path.join(arcname, f), + recursive, exclude, filter) + + else: + self.addfile(tarinfo) + + def addfile(self, tarinfo, fileobj=None): + """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is + given, tarinfo.size bytes are read from it and added to the archive. + You can create TarInfo objects using gettarinfo(). + On Windows platforms, `fileobj' should always be opened with mode + 'rb' to avoid irritation about the file size. + """ + self._check("aw") + + tarinfo = copy.copy(tarinfo) + + buf = tarinfo.tobuf(self.format, self.encoding, self.errors) + self.fileobj.write(buf) + self.offset += len(buf) + + # If there's data to follow, append it. + if fileobj is not None: + copyfileobj(fileobj, self.fileobj, tarinfo.size) + blocks, remainder = divmod(tarinfo.size, BLOCKSIZE) + if remainder > 0: + self.fileobj.write(NUL * (BLOCKSIZE - remainder)) + blocks += 1 + self.offset += blocks * BLOCKSIZE + + self.members.append(tarinfo) + + def extractall(self, path=".", members=None): + """Extract all members from the archive to the current working + directory and set owner, modification time and permissions on + directories afterwards. `path' specifies a different directory + to extract to. `members' is optional and must be a subset of the + list returned by getmembers(). + """ + directories = [] + + if members is None: + members = self + + for tarinfo in members: + if tarinfo.isdir(): + # Extract directories with a safe mode. + directories.append(tarinfo) + tarinfo = copy.copy(tarinfo) + tarinfo.mode = 0700 + self.extract(tarinfo, path) + + # Reverse sort directories. + directories.sort(key=operator.attrgetter('name')) + directories.reverse() + + # Set correct owner, mtime and filemode on directories. + for tarinfo in directories: + dirpath = os.path.join(path, tarinfo.name) + try: + self.chown(tarinfo, dirpath) + self.utime(tarinfo, dirpath) + self.chmod(tarinfo, dirpath) + except ExtractError, e: + if self.errorlevel > 1: + raise + else: + self._dbg(1, "tarfile: %s" % e) + + def extract(self, member, path=""): + """Extract a member from the archive to the current working directory, + using its full name. Its file information is extracted as accurately + as possible. `member' may be a filename or a TarInfo object. You can + specify a different directory using `path'. + """ + self._check("r") + + if isinstance(member, basestring): + tarinfo = self.getmember(member) + else: + tarinfo = member + + # Prepare the link target for makelink(). + if tarinfo.islnk(): + tarinfo._link_target = os.path.join(path, tarinfo.linkname) + + try: + self._extract_member(tarinfo, os.path.join(path, tarinfo.name)) + except EnvironmentError, e: + if self.errorlevel > 0: + raise + else: + if e.filename is None: + self._dbg(1, "tarfile: %s" % e.strerror) + else: + self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) + except ExtractError, e: + if self.errorlevel > 1: + raise + else: + self._dbg(1, "tarfile: %s" % e) + + def extractfile(self, member): + """Extract a member from the archive as a file object. `member' may be + a filename or a TarInfo object. If `member' is a regular file, a + file-like object is returned. If `member' is a link, a file-like + object is constructed from the link's target. If `member' is none of + the above, None is returned. + The file-like object is read-only and provides the following + methods: read(), readline(), readlines(), seek() and tell() + """ + self._check("r") + + if isinstance(member, basestring): + tarinfo = self.getmember(member) + else: + tarinfo = member + + if tarinfo.isreg(): + return self.fileobject(self, tarinfo) + + elif tarinfo.type not in SUPPORTED_TYPES: + # If a member's type is unknown, it is treated as a + # regular file. + return self.fileobject(self, tarinfo) + + elif tarinfo.islnk() or tarinfo.issym(): + if isinstance(self.fileobj, _Stream): + # A small but ugly workaround for the case that someone tries + # to extract a (sym)link as a file-object from a non-seekable + # stream of tar blocks. + raise StreamError("cannot extract (sym)link as file object") + else: + # A (sym)link's file object is its target's file object. + return self.extractfile(self._find_link_target(tarinfo)) + else: + # If there's no data associated with the member (directory, chrdev, + # blkdev, etc.), return None instead of a file object. + return None + + def _extract_member(self, tarinfo, targetpath): + """Extract the TarInfo object tarinfo to a physical + file called targetpath. + """ + # Fetch the TarInfo object for the given name + # and build the destination pathname, replacing + # forward slashes to platform specific separators. + targetpath = targetpath.rstrip("/") + targetpath = targetpath.replace("/", os.sep) + + # Create all upper directories. + upperdirs = os.path.dirname(targetpath) + if upperdirs and not os.path.exists(upperdirs): + # Create directories that are not part of the archive with + # default permissions. + os.makedirs(upperdirs) + + if tarinfo.islnk() or tarinfo.issym(): + self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) + else: + self._dbg(1, tarinfo.name) + + if tarinfo.isreg(): + self.makefile(tarinfo, targetpath) + elif tarinfo.isdir(): + self.makedir(tarinfo, targetpath) + elif tarinfo.isfifo(): + self.makefifo(tarinfo, targetpath) + elif tarinfo.ischr() or tarinfo.isblk(): + self.makedev(tarinfo, targetpath) + elif tarinfo.islnk() or tarinfo.issym(): + self.makelink(tarinfo, targetpath) + elif tarinfo.type not in SUPPORTED_TYPES: + self.makeunknown(tarinfo, targetpath) + else: + self.makefile(tarinfo, targetpath) + + self.chown(tarinfo, targetpath) + if not tarinfo.issym(): + self.chmod(tarinfo, targetpath) + self.utime(tarinfo, targetpath) + + #-------------------------------------------------------------------------- + # Below are the different file methods. They are called via + # _extract_member() when extract() is called. They can be replaced in a + # subclass to implement other functionality. + + def makedir(self, tarinfo, targetpath): + """Make a directory called targetpath. + """ + try: + # Use a safe mode for the directory, the real mode is set + # later in _extract_member(). + os.mkdir(targetpath, 0700) + except EnvironmentError, e: + if e.errno != errno.EEXIST: + raise + + def makefile(self, tarinfo, targetpath): + """Make a file called targetpath. + """ + source = self.extractfile(tarinfo) + target = bltn_open(targetpath, "wb") + copyfileobj(source, target) + source.close() + target.close() + + def makeunknown(self, tarinfo, targetpath): + """Make a file from a TarInfo object with an unknown type + at targetpath. + """ + self.makefile(tarinfo, targetpath) + self._dbg(1, "tarfile: Unknown file type %r, " \ + "extracted as regular file." % tarinfo.type) + + def makefifo(self, tarinfo, targetpath): + """Make a fifo called targetpath. + """ + if hasattr(os, "mkfifo"): + os.mkfifo(targetpath) + else: + raise ExtractError("fifo not supported by system") + + def makedev(self, tarinfo, targetpath): + """Make a character or block device called targetpath. + """ + if not hasattr(os, "mknod") or not hasattr(os, "makedev"): + raise ExtractError("special devices not supported by system") + + mode = tarinfo.mode + if tarinfo.isblk(): + mode |= stat.S_IFBLK + else: + mode |= stat.S_IFCHR + + os.mknod(targetpath, mode, + os.makedev(tarinfo.devmajor, tarinfo.devminor)) + + def makelink(self, tarinfo, targetpath): + """Make a (symbolic) link called targetpath. If it cannot be created + (platform limitation), we try to make a copy of the referenced file + instead of a link. + """ + if hasattr(os, "symlink") and hasattr(os, "link"): + # For systems that support symbolic and hard links. + if tarinfo.issym(): + if os.path.lexists(targetpath): + os.unlink(targetpath) + os.symlink(tarinfo.linkname, targetpath) + else: + # See extract(). + if os.path.exists(tarinfo._link_target): + if os.path.lexists(targetpath): + os.unlink(targetpath) + os.link(tarinfo._link_target, targetpath) + else: + self._extract_member(self._find_link_target(tarinfo), targetpath) + else: + try: + self._extract_member(self._find_link_target(tarinfo), targetpath) + except KeyError: + raise ExtractError("unable to resolve link inside archive") + + def chown(self, tarinfo, targetpath): + """Set owner of targetpath according to tarinfo. + """ + if pwd and hasattr(os, "geteuid") and os.geteuid() == 0: + # We have to be root to do so. + try: + g = grp.getgrnam(tarinfo.gname)[2] + except KeyError: + try: + g = grp.getgrgid(tarinfo.gid)[2] + except KeyError: + g = os.getgid() + try: + u = pwd.getpwnam(tarinfo.uname)[2] + except KeyError: + try: + u = pwd.getpwuid(tarinfo.uid)[2] + except KeyError: + u = os.getuid() + try: + if tarinfo.issym() and hasattr(os, "lchown"): + os.lchown(targetpath, u, g) + else: + if sys.platform != "os2emx": + os.chown(targetpath, u, g) + except EnvironmentError, e: + raise ExtractError("could not change owner") + + def chmod(self, tarinfo, targetpath): + """Set file permissions of targetpath according to tarinfo. + """ + if hasattr(os, 'chmod'): + try: + os.chmod(targetpath, tarinfo.mode) + except EnvironmentError, e: + raise ExtractError("could not change mode") + + def utime(self, tarinfo, targetpath): + """Set modification time of targetpath according to tarinfo. + """ + if not hasattr(os, 'utime'): + return + try: + os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime)) + except EnvironmentError, e: + raise ExtractError("could not change modification time") + + #-------------------------------------------------------------------------- + def next(self): + """Return the next member of the archive as a TarInfo object, when + TarFile is opened for reading. Return None if there is no more + available. + """ + self._check("ra") + if self.firstmember is not None: + m = self.firstmember + self.firstmember = None + return m + + # Read the next block. + self.fileobj.seek(self.offset) + tarinfo = None + while True: + try: + tarinfo = self.tarinfo.fromtarfile(self) + except EOFHeaderError, e: + if self.ignore_zeros: + self._dbg(2, "0x%X: %s" % (self.offset, e)) + self.offset += BLOCKSIZE + continue + except InvalidHeaderError, e: + if self.ignore_zeros: + self._dbg(2, "0x%X: %s" % (self.offset, e)) + self.offset += BLOCKSIZE + continue + elif self.offset == 0: + raise ReadError(str(e)) + except EmptyHeaderError: + if self.offset == 0: + raise ReadError("empty file") + except TruncatedHeaderError, e: + if self.offset == 0: + raise ReadError(str(e)) + except SubsequentHeaderError, e: + raise ReadError(str(e)) + break + + if tarinfo is not None: + self.members.append(tarinfo) + else: + self._loaded = True + + return tarinfo + + #-------------------------------------------------------------------------- + # Little helper methods: + + def _getmember(self, name, tarinfo=None, normalize=False): + """Find an archive member by name from bottom to top. + If tarinfo is given, it is used as the starting point. + """ + # Ensure that all members have been loaded. + members = self.getmembers() + + # Limit the member search list up to tarinfo. + if tarinfo is not None: + members = members[:members.index(tarinfo)] + + if normalize: + name = os.path.normpath(name) + + for member in reversed(members): + if normalize: + member_name = os.path.normpath(member.name) + else: + member_name = member.name + + if name == member_name: + return member + + def _load(self): + """Read through the entire archive file and look for readable + members. + """ + while True: + tarinfo = self.next() + if tarinfo is None: + break + self._loaded = True + + def _check(self, mode=None): + """Check if TarFile is still open, and if the operation's mode + corresponds to TarFile's mode. + """ + if self.closed: + raise IOError("%s is closed" % self.__class__.__name__) + if mode is not None and self.mode not in mode: + raise IOError("bad operation for mode %r" % self.mode) + + def _find_link_target(self, tarinfo): + """Find the target member of a symlink or hardlink member in the + archive. + """ + if tarinfo.issym(): + # Always search the entire archive. + linkname = os.path.dirname(tarinfo.name) + "/" + tarinfo.linkname + limit = None + else: + # Search the archive before the link, because a hard link is + # just a reference to an already archived file. + linkname = tarinfo.linkname + limit = tarinfo + + member = self._getmember(linkname, tarinfo=limit, normalize=True) + if member is None: + raise KeyError("linkname %r not found" % linkname) + return member + + def __iter__(self): + """Provide an iterator object. + """ + if self._loaded: + return iter(self.members) + else: + return TarIter(self) + + def _dbg(self, level, msg): + """Write debugging output to sys.stderr. + """ + if level <= self.debug: + print >> sys.stderr, msg + + def __enter__(self): + self._check() + return self + + def __exit__(self, type, value, traceback): + if type is None: + self.close() + else: + # An exception occurred. We must not call close() because + # it would try to write end-of-archive blocks and padding. + if not self._extfileobj: + self.fileobj.close() + self.closed = True +# class TarFile + +class TarIter: + """Iterator Class. + + for tarinfo in TarFile(...): + suite... + """ + + def __init__(self, tarfile): + """Construct a TarIter object. + """ + self.tarfile = tarfile + self.index = 0 + def __iter__(self): + """Return iterator object. + """ + return self + def next(self): + """Return the next item using TarFile's next() method. + When all members have been read, set TarFile as _loaded. + """ + # Fix for SF #1100429: Under rare circumstances it can + # happen that getmembers() is called during iteration, + # which will cause TarIter to stop prematurely. + if not self.tarfile._loaded: + tarinfo = self.tarfile.next() + if not tarinfo: + self.tarfile._loaded = True + raise StopIteration + else: + try: + tarinfo = self.tarfile.members[self.index] + except IndexError: + raise StopIteration + self.index += 1 + return tarinfo + +# Helper classes for sparse file support +class _section: + """Base class for _data and _hole. + """ + def __init__(self, offset, size): + self.offset = offset + self.size = size + def __contains__(self, offset): + return self.offset <= offset < self.offset + self.size + +class _data(_section): + """Represent a data section in a sparse file. + """ + def __init__(self, offset, size, realpos): + _section.__init__(self, offset, size) + self.realpos = realpos + +class _hole(_section): + """Represent a hole section in a sparse file. + """ + pass + +class _ringbuffer(list): + """Ringbuffer class which increases performance + over a regular list. + """ + def __init__(self): + self.idx = 0 + def find(self, offset): + idx = self.idx + while True: + item = self[idx] + if offset in item: + break + idx += 1 + if idx == len(self): + idx = 0 + if idx == self.idx: + # End of File + return None + self.idx = idx + return item + +#--------------------------------------------- +# zipfile compatible TarFile class +#--------------------------------------------- +TAR_PLAIN = 0 # zipfile.ZIP_STORED +TAR_GZIPPED = 8 # zipfile.ZIP_DEFLATED +class TarFileCompat: + """TarFile class compatible with standard module zipfile's + ZipFile class. + """ + def __init__(self, file, mode="r", compression=TAR_PLAIN): + from warnings import warnpy3k + warnpy3k("the TarFileCompat class has been removed in Python 3.0", + stacklevel=2) + if compression == TAR_PLAIN: + self.tarfile = TarFile.taropen(file, mode) + elif compression == TAR_GZIPPED: + self.tarfile = TarFile.gzopen(file, mode) + else: + raise ValueError("unknown compression constant") + if mode[0:1] == "r": + members = self.tarfile.getmembers() + for m in members: + m.filename = m.name + m.file_size = m.size + m.date_time = time.gmtime(m.mtime)[:6] + def namelist(self): + return map(lambda m: m.name, self.infolist()) + def infolist(self): + return filter(lambda m: m.type in REGULAR_TYPES, + self.tarfile.getmembers()) + def printdir(self): + self.tarfile.list() + def testzip(self): + return + def getinfo(self, name): + return self.tarfile.getmember(name) + def read(self, name): + return self.tarfile.extractfile(self.tarfile.getmember(name)).read() + def write(self, filename, arcname=None, compress_type=None): + self.tarfile.add(filename, arcname) + def writestr(self, zinfo, bytes): + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + import calendar + tinfo = TarInfo(zinfo.filename) + tinfo.size = len(bytes) + tinfo.mtime = calendar.timegm(zinfo.date_time) + self.tarfile.addfile(tinfo, StringIO(bytes)) + def close(self): + self.tarfile.close() +#class TarFileCompat + +#-------------------- +# exported functions +#-------------------- +def is_tarfile(name): + """Return True if name points to a tar archive that we + are able to handle, else return False. + """ + try: + t = open(name) + t.close() + return True + except TarError: + return False + +bltn_open = open +open = TarFile.open -- cgit v1.3