From 4bd3e47da02fde08acfada1795cc55170abdb00a Mon Sep 17 00:00:00 2001 From: jvoisin Date: Tue, 16 Aug 2011 18:11:24 +0200 Subject: setup.py now works ! --- cli.py | 2 +- gui.py | 2 +- lib/__init__.py | 1 - lib/archive.py | 289 ----- lib/audio.py | 98 -- lib/bencode/__init__.py | 1 - lib/bencode/bencode.py | 142 -- lib/hachoir_editor/__init__.py | 8 - lib/hachoir_editor/field.py | 69 - lib/hachoir_editor/fieldset.py | 347 ----- lib/hachoir_editor/typed_field.py | 253 ---- lib/images.py | 37 - lib/mat.py | 162 --- lib/misc.py | 63 - lib/office.py | 257 ---- lib/parser.py | 104 -- lib/pdfrw/__init__.py | 14 - lib/pdfrw/pdfcompress.py | 57 - lib/pdfrw/pdfobjects.py | 183 --- lib/pdfrw/pdfreader.py | 213 --- lib/pdfrw/pdftokens.py | 249 ---- lib/pdfrw/pdfwriter.py | 234 ---- lib/tarfile/__init__.py | 1 - lib/tarfile/tarfile.py | 2594 ------------------------------------- mat/__init__.py | 1 + mat/archive.py | 289 +++++ mat/audio.py | 98 ++ mat/images.py | 37 + mat/mat.py | 162 +++ mat/misc.py | 62 + mat/office.py | 280 ++++ mat/parser.py | 104 ++ setup.py | 2 + test/clitest.py | 2 +- test/libtest.py | 2 +- 35 files changed, 1039 insertions(+), 5380 deletions(-) delete mode 100644 lib/__init__.py delete mode 100644 lib/archive.py delete mode 100644 lib/audio.py delete mode 100644 lib/bencode/__init__.py delete mode 100644 lib/bencode/bencode.py delete mode 100644 lib/hachoir_editor/__init__.py delete mode 100644 lib/hachoir_editor/field.py delete mode 100644 lib/hachoir_editor/fieldset.py delete mode 100644 lib/hachoir_editor/typed_field.py delete mode 100644 lib/images.py delete mode 100644 lib/mat.py delete mode 100644 lib/misc.py delete mode 100644 lib/office.py delete mode 100644 lib/parser.py delete mode 100644 lib/pdfrw/__init__.py delete mode 100644 lib/pdfrw/pdfcompress.py delete mode 100644 lib/pdfrw/pdfobjects.py delete mode 100644 lib/pdfrw/pdfreader.py delete mode 100644 lib/pdfrw/pdftokens.py delete mode 100644 lib/pdfrw/pdfwriter.py delete mode 100644 lib/tarfile/__init__.py delete mode 100755 lib/tarfile/tarfile.py create mode 100644 mat/__init__.py create mode 100644 mat/archive.py create mode 100644 mat/audio.py create mode 100644 mat/images.py create mode 100644 mat/mat.py create mode 100644 mat/misc.py create mode 100644 mat/office.py create mode 100644 mat/parser.py diff --git a/cli.py b/cli.py index 7c4c399..804c4cf 100755 --- a/cli.py +++ b/cli.py @@ -9,7 +9,7 @@ import optparse import hachoir_core -from lib import mat +from mat import mat __version__ = '0.1' diff --git a/gui.py b/gui.py index 4dca692..07b8d6c 100644 --- a/gui.py +++ b/gui.py @@ -14,7 +14,7 @@ import logging import os import xml.sax -from lib import mat +from mat import mat __version__ = '0.1' __author__ = 'jvoisin' diff --git a/lib/__init__.py b/lib/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/lib/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/lib/archive.py b/lib/archive.py deleted file mode 100644 index 77db71c..0000000 --- a/lib/archive.py +++ /dev/null @@ -1,289 +0,0 @@ -''' - Take care of archives formats -''' - -import zipfile -import shutil -import os -import logging -import tempfile - -import parser -import mat -from tarfile import tarfile - - -class GenericArchiveStripper(parser.GenericParser): - ''' - Represent a generic archive - ''' - def __init__(self, filename, parser, mime, backup, add2archive): - super(GenericArchiveStripper, self).__init__(filename, parser, mime, - backup, add2archive) - self.compression = '' - self.add2archive = add2archive - self.tempdir = tempfile.mkdtemp() - - def __del__(self): - ''' - Remove the files inside the temp dir, - then remove the temp dir - ''' - for root, dirs, files in os.walk(self.tempdir): - for item in files: - path_file = os.path.join(root, item) - mat.secure_remove(path_file) - shutil.rmtree(self.tempdir) - - def remove_all(self): - ''' - Call _remove_all() with in argument : "normal" - ''' - self._remove_all('normal') - - def remove_all_ugly(self): - ''' - call remove_all() with in argument : "ugly" - ''' - self._remove_all('ugly') - - def _remove_all(self, method): - ''' - Remove all meta, normal way if method is "normal", - else, use the ugly way (with possible data loss) - ''' - raise NotImplementedError - - -class ZipStripper(GenericArchiveStripper): - ''' - Represent a zip file - ''' - def is_file_clean(self, fileinfo): - ''' - Check if a ZipInfo object is clean of metadatas added - by zip itself, independently of the corresponding file metadatas - ''' - if fileinfo.comment is not '': - return False - elif fileinfo.date_time is not 0: - return False - elif fileinfo.create_system is not 0: - return False - elif fileinfo.create_version is not 0: - return False - else: - return True - - def is_clean(self): - ''' - Check if the given file is clean from harmful metadata - ''' - zipin = zipfile.ZipFile(self.filename, 'r') - if zipin.comment != '': - logging.debug('%s has a comment' % self.filename) - return False - for item in zipin.infolist(): - #I have not found a way to remove the crap added by zipfile :/ - #if not self.is_file_clean(item): - # logging.debug('%s from %s has compromizing zipinfo' % - # (item.filename, self.filename)) - # return False - zipin.extract(item, self.tempdir) - name = os.path.join(self.tempdir, item.filename) - if os.path.isfile(name): - try: - cfile = mat.create_class_file(name, False, - self.add2archive) - if not cfile.is_clean(): - return False - except: - #best solution I have found - logging.info('%s\'s fileformat is not supported, or is a \ -harmless format' % item.filename) - _, ext = os.path.splitext(name) - bname = os.path.basename(item.filename) - if ext not in parser.NOMETA: - if bname != 'mimetype' and bname != '.rels': - return False - zipin.close() - return True - - def get_meta(self): - ''' - Return all the metadata of a ZipFile (don't return metadatas - of contained files : should it ?) - ''' - zipin = zipfile.ZipFile(self.filename, 'r') - metadata = {} - for field in zipin.infolist(): - zipmeta = {} - zipmeta['comment'] = field.comment - zipmeta['modified'] = field.date_time - zipmeta['system'] = field.create_system - zipmeta['zip_version'] = field.create_version - metadata[field.filename] = zipmeta - metadata["%s comment" % self.filename] = zipin.comment - zipin.close() - return metadata - - def _remove_all(self, method): - ''' - So far, the zipfile module does not allow to write a ZipInfo - object into a zipfile (and it's a shame !) : so data added - by zipfile itself could not be removed. It's a big concern. - Is shiping a patched version of zipfile.py a good idea ? - ''' - zipin = zipfile.ZipFile(self.filename, 'r') - zipout = zipfile.ZipFile(self.output, 'w', allowZip64=True) - for item in zipin.infolist(): - zipin.extract(item, self.tempdir) - name = os.path.join(self.tempdir, item.filename) - if os.path.isfile(name): - try: - cfile = mat.create_class_file(name, False, - self.add2archive) - if method is 'normal': - cfile.remove_all() - else: - cfile.remove_all_ugly() - logging.debug('Processing %s from %s' % (item.filename, - self.filename)) - zipout.write(name, item.filename) - except: - logging.info('%s\'s format is not supported or harmless' % - item.filename) - _, ext = os.path.splitext(name) - if self.add2archive or ext in parser.NOMETA: - zipout.write(name, item.filename) - zipout.comment = '' - zipin.close() - zipout.close() - logging.info('%s treated' % self.filename) - self.do_backup() - - -class TarStripper(GenericArchiveStripper): - ''' - Represent a tarfile archive - ''' - def _remove(self, current_file): - ''' - remove the meta added by tar itself to the file - ''' - current_file.mtime = 0 - current_file.uid = 0 - current_file.gid = 0 - current_file.uname = '' - current_file.gname = '' - return current_file - - def _remove_all(self, method): - tarin = tarfile.open(self.filename, 'r' + self.compression) - tarout = tarfile.open(self.output, 'w' + self.compression) - for item in tarin.getmembers(): - tarin.extract(item, self.tempdir) - name = os.path.join(self.tempdir, item.name) - if item.type is '0': # is item a regular file ? - #no backup file - try: - cfile = mat.create_class_file(name, False, - self.add2archive) - if method is 'normal': - cfile.remove_all() - else: - cfile.remove_all_ugly() - tarout.add(name, item.name, filter=self._remove) - except: - logging.info('%s\' format is not supported or harmless' % - item.name) - _, ext = os.path.splitext(name) - if self.add2archive or ext in parser.NOMETA: - tarout.add(name, item.name, filter=self._remove) - tarin.close() - tarout.close() - self.do_backup() - - def is_file_clean(self, current_file): - ''' - Check metadatas added by tar - ''' - if current_file.mtime is not 0: - return False - elif current_file.uid is not 0: - return False - elif current_file.gid is not 0: - return False - elif current_file.uname is not '': - return False - elif current_file.gname is not '': - return False - else: - return True - - def is_clean(self): - ''' - Check if the file is clean from harmful metadatas - ''' - tarin = tarfile.open(self.filename, 'r' + self.compression) - for item in tarin.getmembers(): - if not self.is_file_clean(item): - tarin.close() - return False - tarin.extract(item, self.tempdir) - name = os.path.join(self.tempdir, item.name) - if item.type is '0': # is item a regular file ? - try: - class_file = mat.create_class_file(name, - False, self.add2archive) # no backup file - if not class_file.is_clean(): - tarin.close() - return False - except: - logging.error('%s\'s foramt is not supported or harmless' % - item.filename) - _, ext = os.path.splitext(name) - if ext not in parser.NOMETA: - tarin.close() - return False - tarin.close() - return True - - def get_meta(self): - ''' - Return a dict with all the meta of the file - ''' - tarin = tarfile.open(self.filename, 'r' + self.compression) - metadata = {} - for current_file in tarin.getmembers(): - if current_file.type is '0': - if not self.is_file_clean(current_file): # if there is meta - current_meta = {} - current_meta['mtime'] = current_file.mtime - current_meta['uid'] = current_file.uid - current_meta['gid'] = current_file.gid - current_meta['uname'] = current_file.uname - current_meta['gname'] = current_file.gname - metadata[current_file.name] = current_meta - tarin.close() - return metadata - - -class GzipStripper(TarStripper): - ''' - Represent a tar.gz archive - ''' - def __init__(self, filename, parser, mime, backup, add2archive): - super(GzipStripper, self).__init__(filename, parser, mime, backup, - add2archive) - self.compression = ':gz' - - -class Bzip2Stripper(TarStripper): - ''' - Represents a tar.bz2 archive - ''' - def __init__(self, filename, parser, mime, backup, add2archive): - super(Bzip2Stripper, self).__init__(filename, parser, mime, backup, - add2archive) - self.compression = ':bz2' diff --git a/lib/audio.py b/lib/audio.py deleted file mode 100644 index 18daa7e..0000000 --- a/lib/audio.py +++ /dev/null @@ -1,98 +0,0 @@ -''' - Care about audio fileformat -''' -try: - from mutagen.flac import FLAC - from mutagen.oggvorbis import OggVorbis -except ImportError: - pass - - -import parser -import shutil - - -class MpegAudioStripper(parser.GenericParser): - ''' - Represent mpeg audio file (mp3, ...) - ''' - def _should_remove(self, field): - if field.name in ("id3v1", "id3v2"): - return True - else: - return False - - -class OggStripper(parser.GenericParser): - ''' - Represent an ogg vorbis file - ''' - def remove_all(self): - if self.backup is True: - shutil.copy2(self.filename, self.output) - self.filename = self.output - - mfile = OggVorbis(self.filename) - mfile.delete() - mfile.save() - - def is_clean(self): - ''' - Check if the "metadata" block is present in the file - ''' - mfile = OggVorbis(self.filename) - if mfile.tags == []: - return True - else: - return False - - def get_meta(self): - ''' - Return the content of the metadata block if present - ''' - metadata = {} - mfile = OggVorbis(self.filename) - for key, value in mfile.tags: - metadata[key] = value - return metadata - - -class FlacStripper(parser.GenericParser): - ''' - Represent a Flac audio file - ''' - def remove_all(self): - ''' - Remove the "metadata" block from the file - ''' - if self.backup is True: - shutil.copy2(self.filename, self.output) - self.filename = self.output - - mfile = FLAC(self.filename) - mfile.delete() - mfile.clear_pictures() - mfile.save() - - def is_clean(self): - ''' - Check if the "metadata" block is present in the file - ''' - mfile = FLAC(self.filename) - if mfile.tags is None and mfile.pictures == []: - return True - else: - return False - - def get_meta(self): - ''' - Return the content of the metadata block if present - ''' - metadata = {} - mfile = FLAC(self.filename) - if mfile.tags is not None: - for key, value in mfile.tags: - metadata[key] = value - if mfile.pictures != []: - metadata['picture :'] = 'yes' - return metadata diff --git a/lib/bencode/__init__.py b/lib/bencode/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/lib/bencode/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/lib/bencode/bencode.py b/lib/bencode/bencode.py deleted file mode 100644 index 4acf788..0000000 --- a/lib/bencode/bencode.py +++ /dev/null @@ -1,142 +0,0 @@ -# The contents of this file are subject to the BitTorrent Open Source License -# Version 1.1 (the License). You may not copy or use this file, in either -# source code or executable form, except in compliance with the License. You -# may obtain a copy of the License at http://www.bittorrent.com/license/. -# -# Software distributed under the License is distributed on an AS IS basis, -# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -# for the specific language governing rights and limitations under the -# License. - -# Written by Petru Paler -# Modified by Julien (jvoisin) Voisin - -''' - A quick (and also nice) lib to bencode/bdecode torrent files -''' - - -import types - - -class BTFailure(Exception): - '''Custom Exception''' - pass - - -class Bencached(object): - '''Custom type : cached string''' - __slots__ = ['bencoded'] - - def __init__(self, string): - self.bencoded = string - - -def decode_int(x, f): - '''decode an int''' - f += 1 - newf = x.index('e', f) - n = int(x[f:newf]) - if x[f] == '-': - if x[f + 1] == '0': - raise ValueError - elif x[f] == '0' and newf != f + 1: - raise ValueError - return (n, newf + 1) - - -def decode_string(x, f): - '''decode a string''' - colon = x.index(':', f) - n = int(x[f:colon]) - if x[f] == '0' and colon != f + 1: - raise ValueError - colon += 1 - return (x[colon:colon + n], colon + n) - - -def decode_list(x, f): - '''decode a list''' - result = [] - f += 1 - while x[f] != 'e': - v, f = DECODE_FUNC[x[f]](x, f) - result.append(v) - return (result, f + 1) - - -def decode_dict(x, f): - '''decode a dict''' - result = {} - f += 1 - while x[f] != 'e': - k, f = decode_string(x, f) - result[k], f = DECODE_FUNC[x[f]](x, f) - return (result, f + 1) - - -def encode_bool(x, r): - '''bencode a boolean''' - if x: - encode_int(1, r) - else: - encode_int(0, r) - - -def encode_int(x, r): - '''bencode an integer/float''' - r.extend(('i', str(x), 'e')) - - -def encode_list(x, r): - '''bencode a list/tuple''' - r.append('l') - [ENCODE_FUNC[type(item)](item, r) for item in x] - r.append('e') - - -def encode_dict(x, result): - '''bencode a dict''' - result.append('d') - ilist = x.items() - ilist.sort() - for k, v in ilist: - result.extend((str(len(k)), ':', k)) - ENCODE_FUNC[type(v)](v, result) - result.append('e') - - -DECODE_FUNC = {} -DECODE_FUNC.update(dict([(str(x), decode_string) for x in xrange(9)])) -DECODE_FUNC['l'] = decode_list -DECODE_FUNC['d'] = decode_dict -DECODE_FUNC['i'] = decode_int - - -ENCODE_FUNC = {} -ENCODE_FUNC[Bencached] = lambda x, r: r.append(x.bencoded) -ENCODE_FUNC[types.IntType] = encode_int -ENCODE_FUNC[types.LongType] = encode_int -ENCODE_FUNC[types.StringType] = lambda x, r: r.extend((str(len(x)), ':', x)) -ENCODE_FUNC[types.ListType] = encode_list -ENCODE_FUNC[types.TupleType] = encode_list -ENCODE_FUNC[types.DictType] = encode_dict -ENCODE_FUNC[types.BooleanType] = encode_bool - - -def bencode(string): - '''bencode $string''' - table = [] - ENCODE_FUNC[type(string)](string, table) - return ''.join(table) - - -def bdecode(string): - '''decode $string''' - try: - result, lenght = DECODE_FUNC[string[0]](string, 0) - except (IndexError, KeyError, ValueError): - raise BTFailure('Not a valid bencoded string') - if lenght != len(string): - raise BTFailure('Invalid bencoded value (data after valid prefix)') - return result diff --git a/lib/hachoir_editor/__init__.py b/lib/hachoir_editor/__init__.py deleted file mode 100644 index 1835676..0000000 --- a/lib/hachoir_editor/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from field import ( - EditorError, FakeField) -from typed_field import ( - EditableField, EditableBits, EditableBytes, - EditableInteger, EditableString, - createEditableField) -from fieldset import EditableFieldSet, NewFieldSet, createEditor - diff --git a/lib/hachoir_editor/field.py b/lib/hachoir_editor/field.py deleted file mode 100644 index 6b1efe3..0000000 --- a/lib/hachoir_editor/field.py +++ /dev/null @@ -1,69 +0,0 @@ -from hachoir_core.error import HachoirError -from hachoir_core.field import joinPath, MissingField - -class EditorError(HachoirError): - pass - -class FakeField(object): - """ - This class have API looks similar to Field API, but objects don't contain - any value: all values are _computed_ by parent methods. - - Example: FakeField(editor, "abc").size calls editor._getFieldSize("abc"). - """ - is_field_set = False - - def __init__(self, parent, name): - self._parent = parent - self._name = name - - def _getPath(self): - return joinPath(self._parent.path, self._name) - path = property(_getPath) - - def _getName(self): - return self._name - name = property(_getName) - - def _getAddress(self): - return self._parent._getFieldAddress(self._name) - address = property(_getAddress) - - def _getSize(self): - return self._parent.input[self._name].size - size = property(_getSize) - - def _getValue(self): - return self._parent.input[self._name].value - value = property(_getValue) - - def createDisplay(self): - # TODO: Returns new value if field is altered - return self._parent.input[self._name].display - display = property(createDisplay) - - def _getParent(self): - return self._parent - parent = property(_getParent) - - def hasValue(self): - return self._parent.input[self._name].hasValue() - - def __getitem__(self, key): - # TODO: Implement this function! - raise MissingField(self, key) - - def _isAltered(self): - return False - is_altered = property(_isAltered) - - def writeInto(self, output): - size = self.size - addr = self._parent._getFieldInputAddress(self._name) - input = self._parent.input - stream = input.stream - if size % 8: - output.copyBitsFrom(stream, addr, size, input.endian) - else: - output.copyBytesFrom(stream, addr, size//8) - diff --git a/lib/hachoir_editor/fieldset.py b/lib/hachoir_editor/fieldset.py deleted file mode 100644 index cbc12f9..0000000 --- a/lib/hachoir_editor/fieldset.py +++ /dev/null @@ -1,347 +0,0 @@ -from hachoir_core.dict import UniqKeyError -from hachoir_core.field import MissingField, Float32, Float64, FakeArray -from hachoir_core.compatibility import any -from hachoir_core.i18n import _ -from typed_field import createEditableField -from field import EditorError -from collections import deque # Python 2.4 -import weakref # Python 2.1 -import struct - -class EditableFieldSet(object): - MAX_SIZE = (1 << 40) # Arbitrary limit to catch errors - is_field_set = True - - def __init__(self, parent, fieldset): - self._parent = parent - self.input = fieldset # original FieldSet - self._fields = {} # cache of editable fields - self._deleted = set() # Names of deleted fields - self._inserted = {} # Inserted field (name => list of field, - # where name is the name after) - - def array(self, key): - # FIXME: Use cache? - return FakeArray(self, key) - - def _getParent(self): - return self._parent - parent = property(_getParent) - - def _isAltered(self): - if self._inserted: - return True - if self._deleted: - return True - return any(field.is_altered for field in self._fields.itervalues()) - is_altered = property(_isAltered) - - def reset(self): - """ - Reset the field set and the input field set. - """ - for key, field in self._fields.iteritems(): - if not field.is_altered: - del self._fields[key] - self.input.reset() - - def __len__(self): - return len(self.input) \ - - len(self._deleted) \ - + sum( len(new) for new in self._inserted.itervalues() ) - - def __iter__(self): - for field in self.input: - name = field.name - if name in self._inserted: - for newfield in self._inserted[name]: - yield weakref.proxy(newfield) - if name not in self._deleted: - yield self[name] - if None in self._inserted: - for newfield in self._inserted[None]: - yield weakref.proxy(newfield) - - def insertBefore(self, name, *new_fields): - self._insert(name, new_fields, False) - - def insertAfter(self, name, *new_fields): - self._insert(name, new_fields, True) - - def insert(self, *new_fields): - self._insert(None, new_fields, True) - - def _insert(self, key, new_fields, next): - """ - key is the name of the field before which new_fields - will be inserted. If next is True, the fields will be inserted - _after_ this field. - """ - # Set unique field name - for field in new_fields: - if field._name.endswith("[]"): - self.input.setUniqueFieldName(field) - - # Check that there is no duplicate in inserted fields - new_names = list(field.name for field in new_fields) - names_set = set(new_names) - if len(names_set) != len(new_fields): - duplicates = (name for name in names_set if 1 < new_names.count(name)) - raise UniqKeyError(_("Duplicates in inserted fields: %s") % ", ".join(duplicates)) - - # Check that field names are not in input - if self.input: # Write special version for NewFieldSet? - for name in new_names: - if name in self.input and name not in self._deleted: - raise UniqKeyError(_("Field name '%s' already exists") % name) - - # Check that field names are not in inserted fields - for fields in self._inserted.itervalues(): - for field in fields: - if field.name in new_names: - raise UniqKeyError(_("Field name '%s' already exists") % field.name) - - # Input have already inserted field? - if key in self._inserted: - if next: - self._inserted[key].extend( reversed(new_fields) ) - else: - self._inserted[key].extendleft( reversed(new_fields) ) - return - - # Whould like to insert in inserted fields? - if key: - for fields in self._inserted.itervalues(): - names = [item.name for item in fields] - try: - pos = names.index(key) - except ValueError: - continue - if 0 <= pos: - if next: - pos += 1 - fields.rotate(-pos) - fields.extendleft( reversed(new_fields) ) - fields.rotate(pos) - return - - # Get next field. Use None if we are at the end. - if next: - index = self.input[key].index + 1 - try: - key = self.input[index].name - except IndexError: - key = None - - # Check that field names are not in input - if key not in self.input: - raise MissingField(self, key) - - # Insert in original input - self._inserted[key]= deque(new_fields) - - def _getDescription(self): - return self.input.description - description = property(_getDescription) - - def _getStream(self): - # FIXME: This property is maybe a bad idea since address may be differents - return self.input.stream - stream = property(_getStream) - - def _getName(self): - return self.input.name - name = property(_getName) - - def _getEndian(self): - return self.input.endian - endian = property(_getEndian) - - def _getAddress(self): - if self._parent: - return self._parent._getFieldAddress(self.name) - else: - return 0 - address = property(_getAddress) - - def _getAbsoluteAddress(self): - address = self.address - current = self._parent - while current: - address += current.address - current = current._parent - return address - absolute_address = property(_getAbsoluteAddress) - - def hasValue(self): - return False -# return self._parent.input[self.name].hasValue() - - def _getSize(self): - if self.is_altered: - return sum(field.size for field in self) - else: - return self.input.size - size = property(_getSize) - - def _getPath(self): - return self.input.path - path = property(_getPath) - - def _getOriginalField(self, name): - assert name in self.input - return self.input[name] - - def _getFieldInputAddress(self, name): - """ - Absolute address of a field from the input field set. - """ - assert name in self.input - return self.input[name].absolute_address - - def _getFieldAddress(self, name): - """ - Compute relative address of a field. The operation takes care of - deleted and resized fields. - """ - #assert name not in self._deleted - addr = 0 - for field in self: - if field.name == name: - return addr - addr += field.size - raise MissingField(self, name) - - def _getItemByPath(self, path): - if not path[0]: - path = path[1:] - field = self - for name in path: - field = field[name] - return field - - def __contains__(self, name): - try: - field = self[name] - return (field is not None) - except MissingField: - return False - - def __getitem__(self, key): - """ - Create a weak reference to an editable field (EditableField) for the - field with specified name. If the field is removed later, using the - editable field will raise a weakref.ReferenceError exception. - - May raise a MissingField error if the field doesn't exist in original - field set or it has been deleted. - """ - if "/" in key: - return self._getItemByPath(key.split("/")) - if isinstance(key, (int, long)): - raise EditorError("Integer index are not supported") - - if (key in self._deleted) or (key not in self.input): - raise MissingField(self, key) - if key not in self._fields: - field = self.input[key] - if field.is_field_set: - self._fields[key] = createEditableFieldSet(self, field) - else: - self._fields[key] = createEditableField(self, field) - return weakref.proxy(self._fields[key]) - - def __delitem__(self, name): - """ - Remove a field from the field set. May raise an MissingField exception - if the field has already been deleted. - """ - if name in self._deleted: - raise MissingField(self, name) - self._deleted.add(name) - if name in self._fields: - del self._fields[name] - - def writeInto(self, output): - """ - Write the content if this field set into the output stream - (OutputStream). - """ - if not self.is_altered: - # Not altered: just copy bits/bytes - input = self.input - if input.size % 8: - output.copyBitsFrom(input.stream, - input.absolute_address, input.size, input.endian) - else: - output.copyBytesFrom(input.stream, - input.absolute_address, input.size//8) - else: - # Altered: call writeInto() method of each field - realaddr = 0 - for field in self: - field.writeInto(output) - realaddr += field.size - - def _getValue(self): - raise EditorError('Field set "%s" has no value' % self.path) - def _setValue(self, value): - raise EditorError('Field set "%s" value is read only' % self.path) - value = property(_getValue, _setValue, "Value of field") - -class EditableFloat(EditableFieldSet): - _value = None - - def _isAltered(self): - return (self._value is not None) - is_altered = property(_isAltered) - - def writeInto(self, output): - if self._value is not None: - self._write(output) - else: - EditableFieldSet.writeInto(self, output) - - def _write(self, output): - format = self.input.struct_format - raw = struct.pack(format, self._value) - output.writeBytes(raw) - - def _setValue(self, value): - self.parent._is_altered = True - self._value = value - value = property(EditableFieldSet._getValue, _setValue) - -def createEditableFieldSet(parent, field): - cls = field.__class__ - # FIXME: Support Float80 - if cls in (Float32, Float64): - return EditableFloat(parent, field) - else: - return EditableFieldSet(parent, field) - -class NewFieldSet(EditableFieldSet): - def __init__(self, parent, name): - EditableFieldSet.__init__(self, parent, None) - self._name = name - self._endian = parent.endian - - def __iter__(self): - if None in self._inserted: - return iter(self._inserted[None]) - else: - raise StopIteration() - - def _getName(self): - return self._name - name = property(_getName) - - def _getEndian(self): - return self._endian - endian = property(_getEndian) - - is_altered = property(lambda self: True) - -def createEditor(fieldset): - return EditableFieldSet(None, fieldset) - diff --git a/lib/hachoir_editor/typed_field.py b/lib/hachoir_editor/typed_field.py deleted file mode 100644 index 0f0427b..0000000 --- a/lib/hachoir_editor/typed_field.py +++ /dev/null @@ -1,253 +0,0 @@ -from hachoir_core.field import ( - RawBits, Bit, Bits, PaddingBits, - RawBytes, Bytes, PaddingBytes, - GenericString, Character, - isInteger, isString) -from field import FakeField - -class EditableField(FakeField): - """ - Pure virtual class used to write editable field class. - """ - - _is_altered = False - def __init__(self, parent, name, value=None): - FakeField.__init__(self, parent, name) - self._value = value - - def _isAltered(self): - return self._is_altered - is_altered = property(_isAltered) - - def hasValue(self): - return True - - def _computeSize(self): - raise NotImplementedError() - def _getValue(self): - return self._value - def _setValue(self, value): - self._value = value - - def _propGetValue(self): - if self._value is not None: - return self._getValue() - else: - return FakeField._getValue(self) - def _propSetValue(self, value): - self._setValue(value) - self._is_altered = True - value = property(_propGetValue, _propSetValue) - - def _getSize(self): - if self._value is not None: - return self._computeSize() - else: - return FakeField._getSize(self) - size = property(_getSize) - - def _write(self, output): - raise NotImplementedError() - - def writeInto(self, output): - if self._is_altered: - self._write(output) - else: - return FakeField.writeInto(self, output) - -class EditableFixedField(EditableField): - """ - Editable field with fixed size. - """ - - def __init__(self, parent, name, value=None, size=None): - EditableField.__init__(self, parent, name, value) - if size is not None: - self._size = size - else: - self._size = self._parent._getOriginalField(self._name).size - - def _getSize(self): - return self._size - size = property(_getSize) - -class EditableBits(EditableFixedField): - def __init__(self, parent, name, *args): - if args: - if len(args) != 2: - raise TypeError( - "Wrong argument count, EditableBits constructor prototype is: " - "(parent, name, [size, value])") - size = args[0] - value = args[1] - assert isinstance(value, (int, long)) - else: - size = None - value = None - EditableFixedField.__init__(self, parent, name, value, size) - if args: - self._setValue(args[1]) - self._is_altered = True - - def _setValue(self, value): - if not(0 <= value < (1 << self._size)): - raise ValueError("Invalid value, must be in range %s..%s" - % (0, (1 << self._size) - 1)) - self._value = value - - def _write(self, output): - output.writeBits(self._size, self._value, self._parent.endian) - -class EditableBytes(EditableField): - def _setValue(self, value): - if not value: raise ValueError( - "Unable to set empty string to a EditableBytes field") - self._value = value - - def _computeSize(self): - return len(self._value) * 8 - - def _write(self, output): - output.writeBytes(self._value) - -class EditableString(EditableField): - MAX_SIZE = { - "Pascal8": (1 << 8)-1, - "Pascal16": (1 << 16)-1, - "Pascal32": (1 << 32)-1, - } - - def __init__(self, parent, name, *args, **kw): - if len(args) == 2: - value = args[1] - assert isinstance(value, str) # TODO: support Unicode - elif not args: - value = None - else: - raise TypeError( - "Wrong argument count, EditableString constructor prototype is:" - "(parent, name, [format, value])") - EditableField.__init__(self, parent, name, value) - if len(args) == 2: - self._charset = kw.get('charset', None) - self._format = args[0] - if self._format in GenericString.PASCAL_FORMATS: - self._prefix_size = GenericString.PASCAL_FORMATS[self._format] - else: - self._prefix_size = 0 - self._suffix_str = GenericString.staticSuffixStr( - self._format, self._charset, self._parent.endian) - self._is_altered = True - else: - orig = self._parent._getOriginalField(name) - self._charset = orig.charset - self._format = orig.format - self._prefix_size = orig.content_offset - self._suffix_str = orig.suffix_str - - def _setValue(self, value): - size = len(value) - if self._format in self.MAX_SIZE and self.MAX_SIZE[self._format] < size: - raise ValueError("String is too big") - self._value = value - - def _computeSize(self): - return (self._prefix_size + len(self._value) + len(self._suffix_str))*8 - - def _write(self, output): - if self._format in GenericString.SUFFIX_FORMAT: - output.writeBytes(self._value) - output.writeBytes(self._suffix_str) - elif self._format == "fixed": - output.writeBytes(self._value) - else: - assert self._format in GenericString.PASCAL_FORMATS - size = GenericString.PASCAL_FORMATS[self._format] - output.writeInteger(len(self._value), False, size, self._parent.endian) - output.writeBytes(self._value) - -class EditableCharacter(EditableFixedField): - def __init__(self, parent, name, *args): - if args: - if len(args) != 3: - raise TypeError( - "Wrong argument count, EditableCharacter " - "constructor prototype is: (parent, name, [value])") - value = args[0] - if not isinstance(value, str) or len(value) != 1: - raise TypeError("EditableCharacter needs a character") - else: - value = None - EditableFixedField.__init__(self, parent, name, value, 8) - if args: - self._is_altered = True - - def _setValue(self, value): - if not isinstance(value, str) or len(value) != 1: - raise TypeError("EditableCharacter needs a character") - self._value = value - - def _write(self, output): - output.writeBytes(self._value) - -class EditableInteger(EditableFixedField): - VALID_VALUE_SIGNED = { - 8: (-(1 << 8), (1 << 8)-1), - 16: (-(1 << 15), (1 << 15)-1), - 32: (-(1 << 31), (1 << 31)-1), - } - VALID_VALUE_UNSIGNED = { - 8: (0, (1 << 8)-1), - 16: (0, (1 << 16)-1), - 32: (0, (1 << 32)-1) - } - - def __init__(self, parent, name, *args): - if args: - if len(args) != 3: - raise TypeError( - "Wrong argument count, EditableInteger constructor prototype is: " - "(parent, name, [signed, size, value])") - size = args[1] - value = args[2] - assert isinstance(value, (int, long)) - else: - size = None - value = None - EditableFixedField.__init__(self, parent, name, value, size) - if args: - self._signed = args[0] - self._is_altered = True - else: - self._signed = self._parent._getOriginalField(self._name).signed - - def _setValue(self, value): - if self._signed: - valid = self.VALID_VALUE_SIGNED - else: - valid = self.VALID_VALUE_UNSIGNED - minval, maxval = valid[self._size] - if not(minval <= value <= maxval): - raise ValueError("Invalid value, must be in range %s..%s" - % (minval, maxval)) - self._value = value - - def _write(self, output): - output.writeInteger( - self.value, self._signed, self._size//8, self._parent.endian) - -def createEditableField(fieldset, field): - if isInteger(field): - cls = EditableInteger - elif isString(field): - cls = EditableString - elif field.__class__ in (RawBytes, Bytes, PaddingBytes): - cls = EditableBytes - elif field.__class__ in (RawBits, Bits, Bit, PaddingBits): - cls = EditableBits - elif field.__class__ == Character: - cls = EditableCharacter - else: - cls = FakeField - return cls(fieldset, field.name) - diff --git a/lib/images.py b/lib/images.py deleted file mode 100644 index d090015..0000000 --- a/lib/images.py +++ /dev/null @@ -1,37 +0,0 @@ -''' - Takes care about pictures formats -''' - -import parser - - -class JpegStripper(parser.GenericParser): - ''' - represents a jpeg file - ''' - def _should_remove(self, field): - ''' - return True if the field is compromizing - ''' - if field.name.startswith('comment'): - return True - elif field.name in ("photoshop", "exif", "adobe"): - return True - else: - return False - - -class PngStripper(parser.GenericParser): - ''' - represents a png file - ''' - def _should_remove(self, field): - ''' - return True if the field is compromizing - ''' - if field.name.startswith("text["): - return True - elif field.name is "time": - return True - else: - return False diff --git a/lib/mat.py b/lib/mat.py deleted file mode 100644 index fd13287..0000000 --- a/lib/mat.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python - -''' - Metadata anonymisation toolkit library -''' - -import os -import subprocess -import logging -import mimetypes -import xml.sax - -import hachoir_core.cmd_line -import hachoir_parser - -import images -import audio -import office -import archive -import misc - -__version__ = '0.1' -__author__ = 'jvoisin' - -LOGGING_LEVEL = logging.DEBUG - -logging.basicConfig(level=LOGGING_LEVEL) - -STRIPPERS = { - 'application/x-tar': archive.TarStripper, - 'application/x-gzip': archive.GzipStripper, - 'application/x-bzip2': archive.Bzip2Stripper, - 'application/zip': archive.ZipStripper, - 'audio/mpeg': audio.MpegAudioStripper, - 'image/jpeg': images.JpegStripper, - 'image/png': images.PngStripper, - 'application/x-bittorrent': misc.TorrentStripper, - 'application/opendocument': office.OpenDocumentStripper, - 'application/officeopenxml': office.OpenXmlStripper, -} - -try: - import poppler - import cairo - STRIPPERS['application/x-pdf'] = office.PdfStripper - STRIPPERS['application/pdf'] = office.PdfStripper -except ImportError: - print('Unable to import python-poppler and/or python-cairo: no pdf \ - support') - -try: - import mutagen - STRIPPERS['audio/x-flac'] = audio.FlacStripper - STRIPPERS['audio/vorbis'] = audio.OggStripper -except ImportError: - print('unable to import python-mutagen : limited audio format support') - - -class XMLParser(xml.sax.handler.ContentHandler): - ''' - Parse the supported format xml, and return a corresponding - list of dict - ''' - def __init__(self): - self.dict = {} - self.list = [] - self.content, self.key = '', '' - self.between = False - - def startElement(self, name, attrs): - ''' - Called when entering into xml balise - ''' - self.between = True - self.key = name - self.content = '' - - def endElement(self, name): - ''' - Called when exiting a xml balise - ''' - if name == 'format': # exiting a fileformat section - self.list.append(self.dict.copy()) - self.dict.clear() - else: - content = self.content.replace('\s', ' ') - self.dict[self.key] = content - self.between = False - - def characters(self, characters): - ''' - Concatenate the content between opening and closing balises - ''' - if self.between is True: - self.content += characters - - -def secure_remove(filename): - ''' - securely remove the file - ''' - removed = False - try: - subprocess.call('shred --remove %s' % filename, shell=True) - removed = True - except: - logging.error('Unable to securely remove %s' % filename) - - if removed is False: - try: - os.remove(filename) - except: - logging.error('Unable to remove %s' % filename) - - -def is_secure(filename): - ''' - Prevent shell injection - ''' - if not(os.path.isfile(filename)): # check if the file exist - logging.error('%s is not a valid file' % filename) - return False - else: - return True - - -def create_class_file(name, backup, add2archive): - ''' - return a $FILETYPEStripper() class, - corresponding to the filetype of the given file - ''' - if not is_secure(name): - return - - filename = '' - try: - filename = hachoir_core.cmd_line.unicodeFilename(name) - except TypeError: # get rid of "decoding Unicode is not supported" - filename = name - - parser = hachoir_parser.createParser(filename) - if not parser: - logging.info('Unable to parse %s' % filename) - return - - mime = parser.mime_type - - if mime == 'application/zip': # some formats are zipped stuff - mime = mimetypes.guess_type(name)[0] - - if mime.startswith('application/vnd.oasis.opendocument'): - mime = 'application/opendocument' # opendocument fileformat - elif mime.startswith('application/vnd.openxmlformats-officedocument'): - mime = 'application/officeopenxml' # office openxml - - try: - stripper_class = STRIPPERS[mime] - except KeyError: - logging.info('Don\'t have stripper for %s format' % mime) - return - - return stripper_class(filename, parser, mime, backup, add2archive) diff --git a/lib/misc.py b/lib/misc.py deleted file mode 100644 index fdf53b2..0000000 --- a/lib/misc.py +++ /dev/null @@ -1,63 +0,0 @@ -''' - Care about misc formats -''' - -import parser - -from bencode import bencode -#import bencode - - -class TorrentStripper(parser.GenericParser): - ''' - Represent a torrent file with the help - of the bencode lib from Petru Paler - ''' - def __init__(self, filename, parser, mime, backup, add2archive): - super(TorrentStripper, self).__init__(filename, parser, mime, - backup, add2archive) - self.fields = ['comment', 'creation date', 'created by'] - - def is_clean(self): - ''' - Check if the file is clean from harmful metadatas - ''' - with open(self.filename, 'r') as f: - decoded = bencode.bdecode(f.read()) - for key in self.fields: - try: - if decoded[key] != '': - return False - except: - pass - return True - - def get_meta(self): - ''' - Return a dict with all the meta of the file - ''' - metadata = {} - with open(self.filename, 'r') as f: - decoded = bencode.bdecode(f.read()) - for key in self.fields: - try: - if decoded[key] != '': - metadata[key] = decoded[key] - except: - pass - return metadata - - def remove_all(self): - ''' - Remove all the files that are compromizing - ''' - with open(self.filename, 'r') as f: - decoded = bencode.bdecode(f.read()) - for key in self.fields: - try: - decoded[key] = '' - except: - pass - with open(self.output, 'w') as f: # encode the decoded torrent - f.write(bencode.bencode(decoded)) # and write it in self.output - self.do_backup() diff --git a/lib/office.py b/lib/office.py deleted file mode 100644 index 33af48e..0000000 --- a/lib/office.py +++ /dev/null @@ -1,257 +0,0 @@ -''' - Care about office's formats -''' - -import os -import logging -import zipfile -import fileinput - -try: - import cairo - import poppler -except ImportError: - pass - -import mat -import parser -import archive -import pdfrw - - -class OpenDocumentStripper(archive.GenericArchiveStripper): - ''' - An open document file is a zip, with xml file into. - The one that interest us is meta.xml - ''' - - def get_meta(self): - ''' - Return a dict with all the meta of the file by - trying to read the meta.xml file. - ''' - zipin = zipfile.ZipFile(self.filename, 'r') - metadata = {} - try: - content = zipin.read('meta.xml') - zipin.close() - metadata[self.filename] = 'harful meta' - except KeyError: # no meta.xml file found - logging.debug('%s has no opendocument metadata' % self.filename) - return metadata - - def _remove_all(self, method): - ''' - FIXME ? - There is a patch implementing the Zipfile.remove() - method here : http://bugs.python.org/issue6818 - ''' - zipin = zipfile.ZipFile(self.filename, 'r') - zipout = zipfile.ZipFile(self.output, 'w', allowZip64=True) - - for item in zipin.namelist(): - name = os.path.join(self.tempdir, item) - _, ext = os.path.splitext(name) - - if item.endswith('manifest.xml'): - # contain the list of all files present in the archive - zipin.extract(item, self.tempdir) - for line in fileinput.input(name, inplace=1): - #remove the line which contains "meta.xml" - line = line.strip() - if not 'meta.xml' in line: - print line - zipout.write(name, item) - - elif ext in parser.NOMETA or item == 'mimetype': - #keep NOMETA files, and the "manifest" file - if item != 'meta.xml': # contains the metadata - zipin.extract(item, self.tempdir) - zipout.write(name, item) - - else: - zipin.extract(item, self.tempdir) - if os.path.isfile(name): - try: - cfile = mat.create_class_file(name, False, - self.add2archive) - if method == 'normal': - cfile.remove_all() - else: - cfile.remove_all_ugly() - logging.debug('Processing %s from %s' % (item, - self.filename)) - zipout.write(name, item) - except: - logging.info('%s\' fileformat is not supported' % item) - if self.add2archive: - zipout.write(name, item) - zipout.comment = '' - logging.info('%s treated' % self.filename) - zipin.close() - zipout.close() - self.do_backup() - - def is_clean(self): - ''' - Check if the file is clean from harmful metadatas - ''' - zipin = zipfile.ZipFile(self.filename, 'r') - try: - zipin.getinfo('meta.xml') - except KeyError: # no meta.xml in the file - czf = archive.ZipStripper(self.filename, self.parser, - 'application/zip', self.backup, self.add2archive) - if czf.is_clean(): - zipin.close() - return True - zipin.close() - return False - - -class PdfStripper(parser.GenericParser): - ''' - Represent a pdf file - ''' - def __init__(self, filename, parser, mime, backup, add2archive): - super(PdfStripper, self).__init__(filename, parser, mime, backup, - add2archive) - uri = 'file://' + os.path.abspath(self.filename) - self.password = None - self.document = poppler.document_new_from_file(uri, self.password) - self.meta_list = ('title', 'author', 'subject', 'keywords', 'creator', - 'producer', 'creation-date', 'mod-date', 'metadata') - - def is_clean(self): - ''' - Check if the file is clean from harmful metadatas - ''' - for key in self.meta_list: - if key == 'creation-date' or key == 'mod-date': - if self.document.get_property(key) != -1: - return False - elif self.document.get_property(key) is not None and \ - self.document.get_property(key) != '': - return False - return True - - def remove_all(self): - ''' - Opening the pdf with poppler, then doing a render - on a cairo pdfsurface for each pages. - Thanks to Lunar^for the idea. - http://cairographics.org/documentation/pycairo/2/ - python-poppler is not documented at all : have fun ;) - ''' - page = self.document.get_page(0) - page_width, page_height = page.get_size() - surface = cairo.PDFSurface(self.output, page_width, page_height) - context = cairo.Context(surface) # context draws on the surface - logging.debug('Pdf rendering of %s' % self.filename) - for pagenum in xrange(self.document.get_n_pages()): - page = self.document.get_page(pagenum) - context.translate(0, 0) - page.render(context) # render the page on context - context.show_page() # draw context on surface - surface.finish() - - #For now, poppler cannot write meta, so we must use pdfrw - logging.debug('Removing %s\'s superficial metadata' % self.filename) - trailer = pdfrw.PdfReader(self.output) - trailer.Info.Producer = trailer.Info.Creator = None - writer = pdfrw.PdfWriter() - writer.trailer = trailer - writer.write(self.output) - self.do_backup() - - def get_meta(self): - ''' - Return a dict with all the meta of the file - ''' - metadata = {} - for key in self.meta_list: - if key == 'creation-date' or key == 'mod-date': - #creation and modification are set to -1 - if self.document.get_property(key) != -1: - metadata[key] = self.document.get_property(key) - elif self.document.get_property(key) is not None and \ - self.document.get_property(key) != '': - metadata[key] = self.document.get_property(key) - return metadata - - -class OpenXmlStripper(archive.GenericArchiveStripper): - ''' - Represent an office openxml document, which is like - an opendocument format, with some tricky stuff added. - It contains mostly xml, but can have media blobs, crap, ... - (I don't like this format.) - ''' - def _remove_all(self, method): - ''' - FIXME ? - There is a patch implementing the Zipfile.remove() - method here : http://bugs.python.org/issue6818 - ''' - zipin = zipfile.ZipFile(self.filename, 'r') - zipout = zipfile.ZipFile(self.output, 'w', - allowZip64=True) - for item in zipin.namelist(): - name = os.path.join(self.tempdir, item) - _, ext = os.path.splitext(name) - if item.startswith('docProps/'): # metadatas - pass - elif ext in parser.NOMETA or item == '.rels': - #keep parser.NOMETA files, and the file named ".rels" - zipin.extract(item, self.tempdir) - zipout.write(name, item) - else: - zipin.extract(item, self.tempdir) - if os.path.isfile(name): # don't care about folders - try: - cfile = mat.create_class_file(name, False, - self.add2archive) - if method == 'normal': - cfile.remove_all() - else: - cfile.remove_all_ugly() - logging.debug('Processing %s from %s' % (item, - self.filename)) - zipout.write(name, item) - except: - logging.info('%s\' fileformat is not supported' % item) - if self.add2archive: - zipout.write(name, item) - zipout.comment = '' - logging.info('%s treated' % self.filename) - zipin.close() - zipout.close() - self.do_backup() - - def is_clean(self): - ''' - Check if the file is clean from harmful metadatas - ''' - zipin = zipfile.ZipFile(self.filename, 'r') - for item in zipin.namelist(): - if item.startswith('docProps/'): - return False - zipin.close() - czf = archive.ZipStripper(self.filename, self.parser, - 'application/zip', self.backup, self.add2archive) - if not czf.is_clean(): - return False - else: - return True - - def get_meta(self): - ''' - Return a dict with all the meta of the file - ''' - zipin = zipfile.ZipFile(self.filename, 'r') - metadata = {} - for item in zipin.namelist(): - if item.startswith('docProps/'): - metadata[item] = 'harmful content' - zipin.close() - return metadata diff --git a/lib/parser.py b/lib/parser.py deleted file mode 100644 index 58dd7fa..0000000 --- a/lib/parser.py +++ /dev/null @@ -1,104 +0,0 @@ -''' - Parent class of all parser -''' - -import hachoir_core -import hachoir_editor - -import os - -import mat - -NOMETA = ('.bmp', '.rdf', '.txt', '.xml', '.rels') -#bmp : image -#rdf : text -#txt : plain text -#xml : formated text -#rels : openxml foramted text - - -class GenericParser(object): - ''' - Parent class of all parsers - ''' - def __init__(self, filename, parser, mime, backup, add2archive): - self.filename = '' - self.parser = parser - self.mime = mime - self.backup = backup - self.editor = hachoir_editor.createEditor(parser) - self.realname = filename - try: - self.filename = hachoir_core.cmd_line.unicodeFilename(filename) - except TypeError: # get rid of "decoding Unicode is not supported" - self.filename = filename - basename, ext = os.path.splitext(filename) - self.output = basename + '.cleaned' + ext - self.basename = os.path.basename(filename) # only filename - - def is_clean(self): - ''' - Check if the file is clean from harmful metadatas - ''' - for field in self.editor: - if self._should_remove(field): - return False - return True - - def remove_all(self): - ''' - Remove all the files that are compromizing - ''' - for field in self.editor: - if self._should_remove(field): - self._remove(field.name) - hachoir_core.field.writeIntoFile(self.editor, self.output) - self.do_backup() - - def remove_all_ugly(self): - ''' - If the remove_all() is not efficient enough, - this method is implemented : - It is efficient, but destructive. - In a perfect world, with nice fileformat, - this method would not exist. - ''' - self.remove_all() - - def _remove(self, field): - ''' - Delete the given field - ''' - del self.editor[field] - - def get_meta(self): - ''' - Return a dict with all the meta of the file - ''' - metadata = {} - for field in self.editor: - if self._should_remove(field): - try: - metadata[field.name] = field.value - except: - metadata[field.name] = 'harmful content' - return metadata - - def _should_remove(self, key): - ''' - return True if the field is compromizing - abstract method - ''' - raise NotImplementedError - - def do_backup(self): - ''' - Do a backup of the file if asked, - and change his creation/access date - ''' - if self.backup is True: - os.utime(self.output, (0, 0)) - else: - mat.secure_remove(self.filename) - os.rename(self.output, self.filename) - os.utime(self.filename, (0, 0)) diff --git a/lib/pdfrw/__init__.py b/lib/pdfrw/__init__.py deleted file mode 100644 index 26e8c73..0000000 --- a/lib/pdfrw/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# A part of pdfrw (pdfrw.googlecode.com) -# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas -# MIT license -- See LICENSE.txt for details - -from pdfwriter import PdfWriter -from pdfreader import PdfReader -from pdfobjects import PdfObject, PdfName, PdfArray, PdfDict, IndirectPdfDict, PdfString -from pdftokens import PdfTokens - -# Add a tiny bit of compatibility to pyPdf - -PdfFileReader = PdfReader -PdfFileWriter = PdfWriter - diff --git a/lib/pdfrw/pdfcompress.py b/lib/pdfrw/pdfcompress.py deleted file mode 100644 index 1c11970..0000000 --- a/lib/pdfrw/pdfcompress.py +++ /dev/null @@ -1,57 +0,0 @@ -# A part of pdfrw (pdfrw.googlecode.com) -# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas -# MIT license -- See LICENSE.txt for details - -''' -Currently, this sad little file only knows how to decompress -using the flate (zlib) algorithm. Maybe more later, but it's -not a priority for me... -''' - -from __future__ import generators - -try: - set -except NameError: - from sets import Set as set - -import zlib -from pdfobjects import PdfDict, PdfName - - -def streamobjects(mylist): - for obj in mylist: - if isinstance(obj, PdfDict) and obj.stream is not None: - yield obj - -def uncompress(mylist, warnings=set()): - flate = PdfName.FlateDecode - for obj in streamobjects(mylist): - ftype = obj.Filter - if ftype is None: - continue - if isinstance(ftype, list) and len(ftype) == 1: - # todo: multiple filters - ftype = ftype[0] - parms = obj.DecodeParms - if ftype != flate or parms is not None: - msg = 'Not decompressing: cannot use filter %s with parameters %s' % (repr(ftype), repr(parms)) - if msg not in warnings: - warnings.add(msg) - print msg - else: - obj.stream = zlib.decompress(obj.stream) - obj.Filter = None - -def compress(mylist): - flate = PdfName.FlateDecode - for obj in streamobjects(mylist): - ftype = obj.Filter - if ftype is not None: - continue - oldstr = obj.stream - newstr = zlib.compress(oldstr) - if len(newstr) < len(oldstr) + 30: - obj.stream = newstr - obj.Filter = flate - obj.DecodeParms = None diff --git a/lib/pdfrw/pdfobjects.py b/lib/pdfrw/pdfobjects.py deleted file mode 100644 index 08ad825..0000000 --- a/lib/pdfrw/pdfobjects.py +++ /dev/null @@ -1,183 +0,0 @@ -# A part of pdfrw (pdfrw.googlecode.com) -# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas -# MIT license -- See LICENSE.txt for details - -''' -Objects that can occur in PDF files. The most important -objects are arrays and dicts. Either of these can be -indirect or not, and dicts could have an associated -stream. -''' -from __future__ import generators - -try: - set -except NameError: - from sets import Set as set - -import re - -class PdfObject(str): - indirect = False - -class PdfArray(list): - indirect = False - -class PdfName(object): - def __getattr__(self, name): - return self(name) - def __call__(self, name): - return PdfObject('/' + name) - -PdfName = PdfName() - -class PdfString(str): - indirect = False - unescape_dict = {'\\b':'\b', '\\f':'\f', '\\n':'\n', - '\\r':'\r', '\\t':'\t', - '\\\r\n': '', '\\\r':'', '\\\n':'', - '\\\\':'\\', '\\':'', - } - unescape_pattern = r'(\\b|\\f|\\n|\\r|\\t|\\\r\n|\\\r|\\\n|\\[0-9]+|\\)' - unescape_func = re.compile(unescape_pattern).split - - hex_pattern = '([a-fA-F0-9][a-fA-F0-9]|[a-fA-F0-9])' - hex_func = re.compile(hex_pattern).split - - hex_pattern2 = '([a-fA-F0-9][a-fA-F0-9][a-fA-F0-9][a-fA-F0-9]|[a-fA-F0-9][a-fA-F0-9]|[a-fA-F0-9])' - hex_func2 = re.compile(hex_pattern2).split - - hex_funcs = hex_func, hex_func2 - - indirect = False - - def decode_regular(self, remap=chr): - assert self[0] == '(' and self[-1] == ')' - mylist = self.unescape_func(self[1:-1]) - result = [] - unescape = self.unescape_dict.get - for chunk in mylist: - chunk = unescape(chunk, chunk) - if chunk.startswith('\\') and len(chunk) > 1: - value = int(chunk[1:], 8) - # FIXME: TODO: Handle unicode here - if value > 127: - value = 127 - chunk = remap(value) - if chunk: - result.append(chunk) - return ''.join(result) - - def decode_hex(self, remap=chr, twobytes=False): - data = ''.join(self.split()) - data = self.hex_funcs[twobytes](data) - chars = data[1::2] - other = data[0::2] - assert other[0] == '<' and other[-1] == '>' and ''.join(other) == '<>', self - return ''.join([remap(int(x, 16)) for x in chars]) - - def decode(self, remap=chr, twobytes=False): - if self.startswith('('): - return self.decode_regular(remap) - - else: - return self.decode_hex(remap, twobytes) - - def encode(cls, source, usehex=False): - assert not usehex, "Not supported yet" - if isinstance(source, unicode): - source = source.encode('utf-8') - else: - source = str(source) - source = source.replace('\\', '\\\\') - source = source.replace('(', '\\(') - source = source.replace(')', '\\)') - return cls('(' +source + ')') - encode = classmethod(encode) - -class PdfDict(dict): - indirect = False - stream = None - - _special = dict(indirect = ('indirect', False), - stream = ('stream', True), - _stream = ('stream', False), - ) - - def __setitem__(self, name, value): - assert name.startswith('/'), name - if value is not None: - dict.__setitem__(self, name, value) - elif name in self: - del self[name] - - def __init__(self, *args, **kw): - if args: - if len(args) == 1: - args = args[0] - self.update(args) - if isinstance(args, PdfDict): - self.indirect = args.indirect - self._stream = args.stream - for key, value in kw.iteritems(): - setattr(self, key, value) - - def __getattr__(self, name): - return self.get(PdfName(name)) - - def __setattr__(self, name, value): - info = self._special.get(name) - if info is None: - self[PdfName(name)] = value - else: - name, setlen = info - self.__dict__[name] = value - if setlen: - notnone = value is not None - self.Length = notnone and PdfObject(len(value)) or None - - def iteritems(self): - for key, value in dict.iteritems(self): - if value is not None: - assert key.startswith('/'), (key, value) - yield key, value - - def inheritable(self): - ''' Search through ancestors as needed for inheritable - dictionary items - ''' - class Search(object): - def __init__(self, basedict): - self.basedict = basedict - def __getattr__(self, name): - return self[name] - def __getitem__(self, name): - visited = set() - mydict = self.basedict - while 1: - value = getattr(mydict, name) - if value is not None: - return value - myid = id(mydict) - assert myid not in visited - visited.add(myid) - mydict = mydict.Parent - if mydict is None: - return - return Search(self) - inheritable = property(inheritable) - - def private(self): - ''' Allows setting private metadata for use in - processing (not sent to PDF file) - ''' - class Private(object): - pass - - result = Private() - result.__dict__ = self.__dict__ - return result - private = property(private) - -class IndirectPdfDict(PdfDict): - indirect = True diff --git a/lib/pdfrw/pdfreader.py b/lib/pdfrw/pdfreader.py deleted file mode 100644 index 6f57bea..0000000 --- a/lib/pdfrw/pdfreader.py +++ /dev/null @@ -1,213 +0,0 @@ -# A part of pdfrw (pdfrw.googlecode.com) -# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas -# MIT license -- See LICENSE.txt for details - -''' -The PdfReader class reads an entire PDF file into memory and -parses the top-level container objects. (It does not parse -into streams.) The object subclasses PdfDict, and the -document pages are stored in a list in the pages attribute -of the object. -''' - -from pdftokens import PdfTokens -from pdfobjects import PdfDict, PdfArray, PdfName -from pdfcompress import uncompress - -class PdfReader(PdfDict): - - class unresolved: - # Used as a placeholder until we have an object. - pass - - def readindirect(self, objnum, gennum): - ''' Read an indirect object. If it has already - been read, return it from the cache. - ''' - - def setobj(obj): - # Store the new object in the dictionary - # once we have its value - record[1] = obj - - def ordinary(source, setobj, obj): - # Deal with an ordinary (non-array, non-dict) object - setobj(obj) - return obj - - fdata, objnum, gennum = self.fdata, int(objnum), int(gennum) - record = self.indirect_objects[fdata, objnum, gennum] - if record[1] is not self.unresolved: - return record[1] - - # Read the object header and validate it - source = PdfTokens(fdata, record[0]) - objid = source.multiple(3) - assert int(objid[0]) == objnum, objid - assert int(objid[1]) == gennum, objid - assert objid[2] == 'obj', objid - - # Read the object, and call special code if it starts - # an array or dictionary - obj = source.next() - obj = self.special.get(obj, ordinary)(source, setobj, obj) - self.readstream(obj, source) - obj.indirect = True - return obj - - def readstream(obj, source): - ''' Read optional stream following a dictionary - object. - ''' - tok = source.next() - if tok == 'endobj': - return # No stream - - assert isinstance(obj, PdfDict) - assert tok == 'stream', tok - fdata = source.fdata - floc = fdata.rindex(tok, 0, source.floc) + len(tok) - ch = fdata[floc] - if ch == '\r': - floc += 1 - ch = fdata[floc] - assert ch == '\n' - startstream = floc + 1 - endstream = startstream + int(obj.Length) - obj._stream = fdata[startstream:endstream] - source = PdfTokens(fdata, endstream) - endit = source.multiple(2) - if endit != 'endstream endobj'.split(): - # /Length attribute is broken, try to read stream - # anyway disregarding the specified value - # TODO: issue warning here once we have some kind of - # logging - endstream = fdata.index('endstream', startstream) - if fdata[endstream-2:endstream] == '\r\n': - endstream -= 2 - elif fdata[endstream-1] in ['\n', '\r']: - endstream -= 1 - source = PdfTokens(fdata, endstream) - endit = source.multiple(2) - assert endit == 'endstream endobj'.split() - obj.Length = str(endstream-startstream) - obj._stream = fdata[startstream:endstream] - readstream = staticmethod(readstream) - - def readarray(self, source, setobj=lambda x:None, original=None): - special = self.special - result = PdfArray() - setobj(result) - - for value in source: - if value == ']': - break - if value in special: - value = special[value](source) - elif value == 'R': - generation = result.pop() - value = self.readindirect(result.pop(), generation) - result.append(value) - return result - - def readdict(self, source, setobj=lambda x:None, original=None): - special = self.special - result = PdfDict() - setobj(result) - - tok = source.next() - while tok != '>>': - assert tok.startswith('/'), (tok, source.multiple(10)) - key = tok - value = source.next() - if value in special: - value = special[value](source) - tok = source.next() - else: - tok = source.next() - if value.isdigit() and tok.isdigit(): - assert source.next() == 'R' - value = self.readindirect(value, tok) - tok = source.next() - result[key] = value - - return result - - def readxref(fdata): - startloc = fdata.rindex('startxref') - xrefinfo = list(PdfTokens(fdata, startloc, False)) - assert len(xrefinfo) == 3, xrefinfo - assert xrefinfo[0] == 'startxref', xrefinfo[0] - assert xrefinfo[1].isdigit(), xrefinfo[1] - assert xrefinfo[2].rstrip() == '%%EOF', repr(xrefinfo[2]) - return startloc, PdfTokens(fdata, int(xrefinfo[1])) - readxref = staticmethod(readxref) - - def parsexref(self, source): - tok = source.next() - assert tok == 'xref', tok - while 1: - tok = source.next() - if tok == 'trailer': - break - startobj = int(tok) - for objnum in range(startobj, startobj + int(source.next())): - offset = int(source.next()) - generation = int(source.next()) - if source.next() == 'n': - objid = self.fdata, objnum, generation - objval = [offset, self.unresolved] - self.indirect_objects.setdefault(objid, objval) - - pagename = PdfName.Page - pagesname = PdfName.Pages - - def readpages(self, node): - # PDFs can have arbitrarily nested Pages/Page - # dictionary structures. - if node.Type == self.pagename: - return [node] - assert node.Type == self.pagesname, node.Type - result = [] - for node in node.Kids: - result.extend(self.readpages(node)) - return result - - def __init__(self, fname=None, fdata=None, decompress=True): - - if fname is not None: - assert fdata is None - # Allow reading preexisting streams like pyPdf - if hasattr(fname, 'read'): - fdata = fname.read() - else: - f = open(fname, 'rb') - fdata = f.read() - f.close() - - assert fdata is not None - fdata = fdata.rstrip('\00') - self.private.fdata = fdata - - self.private.indirect_objects = {} - self.private.special = {'<<': self.readdict, '[': self.readarray} - - startloc, source = self.readxref(fdata) - self.parsexref(source) - assert source.next() == '<<' - self.update(self.readdict(source)) - assert source.next() == 'startxref' and source.floc > startloc - self.private.pages = self.readpages(self.Root.Pages) - if decompress: - self.uncompress() - - # For compatibility with pyPdf - self.private.numPages = len(self.pages) - - - # For compatibility with pyPdf - def getPage(self, pagenum): - return self.pages[pagenum] - - def uncompress(self): - uncompress([x[1] for x in self.indirect_objects.itervalues()]) diff --git a/lib/pdfrw/pdftokens.py b/lib/pdfrw/pdftokens.py deleted file mode 100644 index 04bd559..0000000 --- a/lib/pdfrw/pdftokens.py +++ /dev/null @@ -1,249 +0,0 @@ -# A part of pdfrw (pdfrw.googlecode.com) -# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas -# MIT license -- See LICENSE.txt for details - -''' -A tokenizer for PDF streams. - -In general, documentation used was "PDF reference", -sixth edition, for PDF version 1.7, dated November 2006. - -''' - -from __future__ import generators - -try: - set -except NameError: - from sets import Set as set - -import re -from pdfobjects import PdfString, PdfObject - -class _PrimitiveTokens(object): - - # Table 3.1, page 50 of reference, defines whitespace - whitespaceset = set('\x00\t\n\f\r ') - - - # Text on page 50 defines delimiter characters - delimiterset = set('()<>{}[]/%') - - # Coalesce contiguous whitespace into a single token - whitespace_pattern = '[%s]+' % ''.join(whitespaceset) - - # In addition to the delimiters, we also use '\', which - # is special in some contexts in PDF. - delimiter_pattern = '\\\\|\\' + '|\\'.join(delimiterset) - - # Dictionary delimiters are '<<' and '>>'. Look for - # these before the single variety. - dictdelim_pattern = r'\<\<|\>\>' - - pattern = '(%s|%s|%s)' % (whitespace_pattern, - dictdelim_pattern, delimiter_pattern) - re_func = re.compile(pattern).finditer - del whitespace_pattern, dictdelim_pattern - del delimiter_pattern, pattern - - def __init__(self, fdata): - - class MyIterator(object): - def next(): - if not tokens: - startloc = self.startloc - for match in next_match[0]: - start = match.start() - end = match.end() - tappend(fdata[start:end]) - if start > startloc: - tappend(fdata[startloc:start]) - self.startloc = end - break - else: - s = fdata[startloc:] - self.startloc = len(fdata) - if s: - tappend(s) - if not tokens: - raise StopIteration - return tpop() - next = staticmethod(next) - - self.fdata = fdata - self.tokens = tokens = [] - self.iterator = iterator = MyIterator() - self.next = iterator.next - self.next_match = next_match = [None] - tappend = tokens.append - tpop = tokens.pop - - def setstart(self, startloc): - self.startloc = startloc - self.next_match[0] = self.re_func(self.fdata, startloc) - - def __iter__(self): - return self.iterator - - def coalesce(self, result): - ''' This function coalesces tokens together up until - the next delimiter or whitespace. - All of the coalesced tokens will either be non-matches, - or will be a matched backslash. We distinguish the - non-matches by the fact that next() will have left - a following match inside self.tokens for the actual match. - ''' - tokens = self.tokens - whitespace = self.whitespaceset - - # Optimized path for usual case -- regular data (not a name string), - # with no escape character, and followed by whitespace. - - if tokens: - token = tokens.pop() - if token != '\\': - if token[0] not in whitespace: - tokens.append(token) - return - result.append(token) - - # Non-optimized path. Either start of a name string received, - # or we just had one escape. - - for token in self: - if tokens: - result.append(token) - token = tokens.pop() - if token != '\\': - if token[0] not in whitespace: - tokens.append(token) - return - result.append(token) - - - def floc(self): - return self.startloc - sum([len(x) for x in self.tokens]) - -class PdfTokens(object): - - def __init__(self, fdata, startloc=0, strip_comments=True): - - def comment(token): - tokens = [token] - for token in primitive: - tokens.append(token) - if token[0] in whitespaceset and ('\n' in token or '\r' in token): - break - return not strip_comments and ''.join(tokens) - - def single(token): - return token - - def regular_string(token): - def escaped(): - escaped = False - i = -2 - while tokens[i] == '\\': - escaped = not escaped - i -= 1 - return escaped - - tokens = [token] - nestlevel = 1 - for token in primitive: - tokens.append(token) - if token in '()' and not escaped(): - nestlevel += token == '(' or -1 - if not nestlevel: - break - else: - assert 0, "Unexpected end of token stream" - return PdfString(''.join(tokens)) - - def hex_string(token): - tokens = [token] - for token in primitive: - tokens.append(token) - if token == '>': - break - while tokens[-2] == '>>': - tokens.append(tokens.pop(-2)) - return PdfString(''.join(tokens)) - - def normal_data(token): - - # Obscure optimization -- we can get here with - # whitespace or regular character data. If we get - # here with whitespace, then there won't be an additional - # token queued up in the primitive object, otherwise there - # will... - if primitive_tokens: #if token[0] not in whitespaceset: - tokens = [token] - primitive.coalesce(tokens) - return PdfObject(''.join(tokens)) - - def name_string(token): - tokens = [token] - primitive.coalesce(tokens) - token = ''.join(tokens) - if '#' in token: - substrs = token.split('#') - substrs.reverse() - tokens = [substrs.pop()] - while substrs: - s = substrs.pop() - tokens.append(chr(int(s[:2], 16))) - tokens.append(s[2:]) - token = ''.join(tokens) - return PdfObject(token) - - def broken(token): - assert 0, token - - dispatch = { - '(': regular_string, - ')': broken, - '<': hex_string, - '>': broken, - '[': single, - ']': single, - '{': single, - '}': single, - '/': name_string, - '%' : comment, - '<<': single, - '>>': single, - }.get - - class MyIterator(object): - def next(): - while not tokens: - token = primitive_next() - token = dispatch(token, normal_data)(token) - if token: - return token - return tokens.pop() - next = staticmethod(next) - - self.primitive = primitive = _PrimitiveTokens(fdata) - self.setstart = primitive.setstart - primitive.setstart(startloc) - self.fdata = fdata - self.strip_comments = strip_comments - self.tokens = tokens = [] - self.iterator = iterator = MyIterator() - self.next = iterator.next - primitive_next = primitive.next - primitive_tokens = primitive.tokens - whitespaceset = _PrimitiveTokens.whitespaceset - - def floc(self): - return self.primitive.floc() - sum([len(x) for x in self.tokens]) - floc = property(floc) - - def __iter__(self): - return self.iterator - - def multiple(self, count): - next = self.next - return [next() for i in range(count)] diff --git a/lib/pdfrw/pdfwriter.py b/lib/pdfrw/pdfwriter.py deleted file mode 100644 index c193843..0000000 --- a/lib/pdfrw/pdfwriter.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python - -# A part of pdfrw (pdfrw.googlecode.com) -# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas -# MIT license -- See LICENSE.txt for details - -''' -The PdfWriter class writes an entire PDF file out to disk. - -The writing process is not at all optimized or organized. - -An instance of the PdfWriter class has two methods: - addpage(page) -and - write(fname) - -addpage() assumes that the pages are part of a valid -tree/forest of PDF objects. -''' - -try: - set -except NameError: - from sets import Set as set - -from pdfobjects import PdfName, PdfArray, PdfDict, IndirectPdfDict, PdfObject, PdfString -from pdfcompress import compress - -debug = False - -class FormatObjects(object): - ''' FormatObjects performs the actual formatting and disk write. - ''' - - def add(self, obj, visited): - ''' Add an object to our list, if it's an indirect - object. Just format it if not. - ''' - # Can't hash dicts, so just hash the object ID - objid = id(obj) - - # Automatically set stream objects to indirect - if isinstance(obj, PdfDict): - indirect = obj.indirect or (obj.stream is not None) - else: - indirect = getattr(obj, 'indirect', False) - - if not indirect: - assert objid not in visited, \ - 'Circular reference encountered in non-indirect object %s' % repr(obj) - visited.add(objid) - result = self.format_obj(obj, visited) - visited.remove(objid) - return result - - objnum = self.indirect_dict.get(objid) - - # If we haven't seen the object yet, we need to - # add it to the indirect object list. - if objnum is None: - objlist = self.objlist - objnum = len(objlist) + 1 - if debug: - print ' Object', objnum, '\r', - objlist.append(None) - self.indirect_dict[objid] = objnum - objlist[objnum-1] = self.format_obj(obj) - return '%s 0 R' % objnum - - def format_array(myarray, formatter): - # Format array data into semi-readable ASCII - if sum([len(x) for x in myarray]) <= 70: - return formatter % ' '.join(myarray) - bigarray = [] - count = 1000000 - for x in myarray: - lenx = len(x) - if lenx + count > 70: - subarray = [] - bigarray.append(subarray) - count = 0 - count += lenx + 1 - subarray.append(x) - return formatter % '\n '.join([' '.join(x) for x in bigarray]) - format_array = staticmethod(format_array) - - def format_obj(self, obj, visited=None): - ''' format PDF object data into semi-readable ASCII. - May mutually recurse with add() -- add() will - return references for indirect objects, and add - the indirect object to the list. - ''' - if visited is None: - visited = set() - if isinstance(obj, PdfArray): - myarray = [self.add(x, visited) for x in obj] - return self.format_array(myarray, '[%s]') - elif isinstance(obj, PdfDict): - if self.compress and obj.stream: - compress([obj]) - myarray = [] - # Jython 2.2.1 has a bug which segfaults when - # sorting subclassed strings, so we un-subclass them. - dictkeys = [str(x) for x in obj.iterkeys()] - dictkeys.sort() - for key in dictkeys: - myarray.append(key) - myarray.append(self.add(obj[key], visited)) - result = self.format_array(myarray, '<<%s>>') - stream = obj.stream - if stream is not None: - result = '%s\nstream\n%s\nendstream' % (result, stream) - return result - elif isinstance(obj, basestring) and not hasattr(obj, 'indirect'): - return PdfString.encode(obj) - else: - return str(obj) - - def dump(cls, f, trailer, version='1.3', compress=True): - self = cls() - self.compress = compress - self.indirect_dict = {} - self.objlist = [] - - # The first format of trailer gets all the information, - # but we throw away the actual trailer formatting. - self.format_obj(trailer) - # Now we know the size, so we update the trailer dict - # and get the formatted data. - trailer.Size = PdfObject(len(self.objlist) + 1) - trailer = self.format_obj(trailer) - - # Now we have all the pieces to write out to the file. - # Keep careful track of the counts while we do it so - # we can correctly build the cross-reference. - - header = '%%PDF-%s\n%%\xe2\xe3\xcf\xd3\n' % version - f.write(header) - offset = len(header) - offsets = [(0, 65535, 'f')] - - for i, x in enumerate(self.objlist): - objstr = '%s 0 obj\n%s\nendobj\n' % (i + 1, x) - offsets.append((offset, 0, 'n')) - offset += len(objstr) - f.write(objstr) - - f.write('xref\n0 %s\n' % len(offsets)) - for x in offsets: - f.write('%010d %05d %s\r\n' % x) - f.write('trailer\n\n%s\nstartxref\n%s\n%%%%EOF\n' % (trailer, offset)) - dump = classmethod(dump) - -class PdfWriter(object): - - _trailer = None - - def __init__(self, version='1.3', compress=True): - self.pagearray = PdfArray() - self.compress = compress - self.version = version - - def addpage(self, page): - self._trailer = None - assert page.Type == PdfName.Page - inheritable = page.inheritable # searches for resources - self.pagearray.append( - IndirectPdfDict( - page, - Resources = inheritable.Resources, - MediaBox = inheritable.MediaBox, - CropBox = inheritable.CropBox, - Rotate = inheritable.Rotate, - ) - ) - return self - - addPage = addpage # for compatibility with pyPdf - - def addpages(self, pagelist): - for page in pagelist: - self.addpage(page) - return self - - def _get_trailer(self): - trailer = self._trailer - if trailer is not None: - return trailer - - # Create the basic object structure of the PDF file - trailer = PdfDict( - Root = IndirectPdfDict( - Type = PdfName.Catalog, - Pages = IndirectPdfDict( - Type = PdfName.Pages, - Count = PdfObject(len(self.pagearray)), - Kids = self.pagearray - ) - ) - ) - # Make all the pages point back to the page dictionary - pagedict = trailer.Root.Pages - for page in pagedict.Kids: - page.Parent = pagedict - self._trailer = trailer - return trailer - - def _set_trailer(self, trailer): - self._trailer = trailer - - trailer = property(_get_trailer, _set_trailer) - - def write(self, fname, trailer=None): - trailer = trailer or self.trailer - - # Dump the data. We either have a filename or a preexisting - # file object. - preexisting = hasattr(fname, 'write') - f = preexisting and fname or open(fname, 'wb') - FormatObjects.dump(f, trailer, self.version, self.compress) - if not preexisting: - f.close() - -if __name__ == '__main__': - debug = True - import pdfreader - x = pdfreader.PdfReader('source.pdf') - y = PdfWriter() - for i, page in enumerate(x.pages): - print ' Adding page', i+1, '\r', - y.addpage(page) - print - y.write('result.pdf') - print diff --git a/lib/tarfile/__init__.py b/lib/tarfile/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/lib/tarfile/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/lib/tarfile/tarfile.py b/lib/tarfile/tarfile.py deleted file mode 100755 index 37c9b92..0000000 --- a/lib/tarfile/tarfile.py +++ /dev/null @@ -1,2594 +0,0 @@ -#! /usr/bin/python2.7 -# -*- coding: iso-8859-1 -*- -#------------------------------------------------------------------- -# tarfile.py -#------------------------------------------------------------------- -# Copyright (C) 2002 Lars Gustäbel -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -"""Read from and write to tar format archives. -""" - -__version__ = "$Revision$" -# $Source$ - -version = "0.9.0" -__author__ = "Lars Gustäbel (lars@gustaebel.de)" -__date__ = "$Date$" -__cvsid__ = "$Id$" -__credits__ = "Gustavo Niemeyer, Niels Gustäbel, Richard Townsend." - -#--------- -# Imports -#--------- -import sys -import os -import shutil -import stat -import errno -import time -import struct -import copy -import re -import operator - -try: - import grp, pwd -except ImportError: - grp = pwd = None - -# from tarfile import * -__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"] - -#--------------------------------------------------------- -# tar constants -#--------------------------------------------------------- -NUL = "\0" # the null character -BLOCKSIZE = 512 # length of processing blocks -RECORDSIZE = BLOCKSIZE * 20 # length of records -GNU_MAGIC = "ustar \0" # magic gnu tar string -POSIX_MAGIC = "ustar\x0000" # magic posix tar string - -LENGTH_NAME = 100 # maximum length of a filename -LENGTH_LINK = 100 # maximum length of a linkname -LENGTH_PREFIX = 155 # maximum length of the prefix field - -REGTYPE = "0" # regular file -AREGTYPE = "\0" # regular file -LNKTYPE = "1" # link (inside tarfile) -SYMTYPE = "2" # symbolic link -CHRTYPE = "3" # character special device -BLKTYPE = "4" # block special device -DIRTYPE = "5" # directory -FIFOTYPE = "6" # fifo special device -CONTTYPE = "7" # contiguous file - -GNUTYPE_LONGNAME = "L" # GNU tar longname -GNUTYPE_LONGLINK = "K" # GNU tar longlink -GNUTYPE_SPARSE = "S" # GNU tar sparse file - -XHDTYPE = "x" # POSIX.1-2001 extended header -XGLTYPE = "g" # POSIX.1-2001 global header -SOLARIS_XHDTYPE = "X" # Solaris extended header - -USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format -GNU_FORMAT = 1 # GNU tar format -PAX_FORMAT = 2 # POSIX.1-2001 (pax) format -DEFAULT_FORMAT = GNU_FORMAT - -#--------------------------------------------------------- -# tarfile constants -#--------------------------------------------------------- -# File types that tarfile supports: -SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE, - SYMTYPE, DIRTYPE, FIFOTYPE, - CONTTYPE, CHRTYPE, BLKTYPE, - GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, - GNUTYPE_SPARSE) - -# File types that will be treated as a regular file. -REGULAR_TYPES = (REGTYPE, AREGTYPE, - CONTTYPE, GNUTYPE_SPARSE) - -# File types that are part of the GNU tar format. -GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, - GNUTYPE_SPARSE) - -# Fields from a pax header that override a TarInfo attribute. -PAX_FIELDS = ("path", "linkpath", "size", "mtime", - "uid", "gid", "uname", "gname") - -# Fields in a pax header that are numbers, all other fields -# are treated as strings. -PAX_NUMBER_FIELDS = { - "atime": float, - "ctime": float, - "mtime": float, - "uid": int, - "gid": int, - "size": int -} - -#--------------------------------------------------------- -# Bits used in the mode field, values in octal. -#--------------------------------------------------------- -S_IFLNK = 0120000 # symbolic link -S_IFREG = 0100000 # regular file -S_IFBLK = 0060000 # block device -S_IFDIR = 0040000 # directory -S_IFCHR = 0020000 # character device -S_IFIFO = 0010000 # fifo - -TSUID = 04000 # set UID on execution -TSGID = 02000 # set GID on execution -TSVTX = 01000 # reserved - -TUREAD = 0400 # read by owner -TUWRITE = 0200 # write by owner -TUEXEC = 0100 # execute/search by owner -TGREAD = 0040 # read by group -TGWRITE = 0020 # write by group -TGEXEC = 0010 # execute/search by group -TOREAD = 0004 # read by other -TOWRITE = 0002 # write by other -TOEXEC = 0001 # execute/search by other - -#--------------------------------------------------------- -# initialization -#--------------------------------------------------------- -ENCODING = sys.getfilesystemencoding() -if ENCODING is None: - ENCODING = sys.getdefaultencoding() - -#--------------------------------------------------------- -# Some useful functions -#--------------------------------------------------------- - -def stn(s, length): - """Convert a python string to a null-terminated string buffer. - """ - return s[:length] + (length - len(s)) * NUL - -def nts(s): - """Convert a null-terminated string field to a python string. - """ - # Use the string up to the first null char. - p = s.find("\0") - if p == -1: - return s - return s[:p] - -def nti(s): - """Convert a number field to a python number. - """ - # There are two possible encodings for a number field, see - # itn() below. - if s[0] != chr(0200): - try: - n = int(nts(s) or "0", 8) - except ValueError: - raise InvalidHeaderError("invalid header") - else: - n = 0L - for i in xrange(len(s) - 1): - n <<= 8 - n += ord(s[i + 1]) - return n - -def itn(n, digits=8, format=DEFAULT_FORMAT): - """Convert a python number to a number field. - """ - # POSIX 1003.1-1988 requires numbers to be encoded as a string of - # octal digits followed by a null-byte, this allows values up to - # (8**(digits-1))-1. GNU tar allows storing numbers greater than - # that if necessary. A leading 0200 byte indicates this particular - # encoding, the following digits-1 bytes are a big-endian - # representation. This allows values up to (256**(digits-1))-1. - if 0 <= n < 8 ** (digits - 1): - s = "%0*o" % (digits - 1, n) + NUL - else: - if format != GNU_FORMAT or n >= 256 ** (digits - 1): - raise ValueError("overflow in number field") - - if n < 0: - # XXX We mimic GNU tar's behaviour with negative numbers, - # this could raise OverflowError. - n = struct.unpack("L", struct.pack("l", n))[0] - - s = "" - for i in xrange(digits - 1): - s = chr(n & 0377) + s - n >>= 8 - s = chr(0200) + s - return s - -def uts(s, encoding, errors): - """Convert a unicode object to a string. - """ - if errors == "utf-8": - # An extra error handler similar to the -o invalid=UTF-8 option - # in POSIX.1-2001. Replace untranslatable characters with their - # UTF-8 representation. - try: - return s.encode(encoding, "strict") - except UnicodeEncodeError: - x = [] - for c in s: - try: - x.append(c.encode(encoding, "strict")) - except UnicodeEncodeError: - x.append(c.encode("utf8")) - return "".join(x) - else: - return s.encode(encoding, errors) - -def calc_chksums(buf): - """Calculate the checksum for a member's header by summing up all - characters except for the chksum field which is treated as if - it was filled with spaces. According to the GNU tar sources, - some tars (Sun and NeXT) calculate chksum with signed char, - which will be different if there are chars in the buffer with - the high bit set. So we calculate two checksums, unsigned and - signed. - """ - unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512])) - signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512])) - return unsigned_chksum, signed_chksum - -def copyfileobj(src, dst, length=None): - """Copy length bytes from fileobj src to fileobj dst. - If length is None, copy the entire content. - """ - if length == 0: - return - if length is None: - shutil.copyfileobj(src, dst) - return - - BUFSIZE = 16 * 1024 - blocks, remainder = divmod(length, BUFSIZE) - for b in xrange(blocks): - buf = src.read(BUFSIZE) - if len(buf) < BUFSIZE: - raise IOError("end of file reached") - dst.write(buf) - - if remainder != 0: - buf = src.read(remainder) - if len(buf) < remainder: - raise IOError("end of file reached") - dst.write(buf) - return - -filemode_table = ( - ((S_IFLNK, "l"), - (S_IFREG, "-"), - (S_IFBLK, "b"), - (S_IFDIR, "d"), - (S_IFCHR, "c"), - (S_IFIFO, "p")), - - ((TUREAD, "r"),), - ((TUWRITE, "w"),), - ((TUEXEC|TSUID, "s"), - (TSUID, "S"), - (TUEXEC, "x")), - - ((TGREAD, "r"),), - ((TGWRITE, "w"),), - ((TGEXEC|TSGID, "s"), - (TSGID, "S"), - (TGEXEC, "x")), - - ((TOREAD, "r"),), - ((TOWRITE, "w"),), - ((TOEXEC|TSVTX, "t"), - (TSVTX, "T"), - (TOEXEC, "x")) -) - -def filemode(mode): - """Convert a file's mode to a string of the form - -rwxrwxrwx. - Used by TarFile.list() - """ - perm = [] - for table in filemode_table: - for bit, char in table: - if mode & bit == bit: - perm.append(char) - break - else: - perm.append("-") - return "".join(perm) - -class TarError(Exception): - """Base exception.""" - pass -class ExtractError(TarError): - """General exception for extract errors.""" - pass -class ReadError(TarError): - """Exception for unreadble tar archives.""" - pass -class CompressionError(TarError): - """Exception for unavailable compression methods.""" - pass -class StreamError(TarError): - """Exception for unsupported operations on stream-like TarFiles.""" - pass -class HeaderError(TarError): - """Base exception for header errors.""" - pass -class EmptyHeaderError(HeaderError): - """Exception for empty headers.""" - pass -class TruncatedHeaderError(HeaderError): - """Exception for truncated headers.""" - pass -class EOFHeaderError(HeaderError): - """Exception for end of file headers.""" - pass -class InvalidHeaderError(HeaderError): - """Exception for invalid headers.""" - pass -class SubsequentHeaderError(HeaderError): - """Exception for missing and invalid extended headers.""" - pass - -#--------------------------- -# internal stream interface -#--------------------------- -class _LowLevelFile: - """Low-level file object. Supports reading and writing. - It is used instead of a regular file object for streaming - access. - """ - - def __init__(self, name, mode): - mode = { - "r": os.O_RDONLY, - "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC, - }[mode] - if hasattr(os, "O_BINARY"): - mode |= os.O_BINARY - self.fd = os.open(name, mode, 0666) - - def close(self): - os.close(self.fd) - - def read(self, size): - return os.read(self.fd, size) - - def write(self, s): - os.write(self.fd, s) - -class _Stream: - """Class that serves as an adapter between TarFile and - a stream-like object. The stream-like object only - needs to have a read() or write() method and is accessed - blockwise. Use of gzip or bzip2 compression is possible. - A stream-like object could be for example: sys.stdin, - sys.stdout, a socket, a tape device etc. - - _Stream is intended to be used only internally. - """ - - def __init__(self, name, mode, comptype, fileobj, bufsize): - """Construct a _Stream object. - """ - self._extfileobj = True - if fileobj is None: - fileobj = _LowLevelFile(name, mode) - self._extfileobj = False - - if comptype == '*': - # Enable transparent compression detection for the - # stream interface - fileobj = _StreamProxy(fileobj) - comptype = fileobj.getcomptype() - - self.name = name or "" - self.mode = mode - self.comptype = comptype - self.fileobj = fileobj - self.bufsize = bufsize - self.buf = "" - self.pos = 0L - self.closed = False - - if comptype == "gz": - try: - import zlib - except ImportError: - raise CompressionError("zlib module is not available") - self.zlib = zlib - self.crc = zlib.crc32("") & 0xffffffffL - if mode == "r": - self._init_read_gz() - else: - self._init_write_gz() - - if comptype == "bz2": - try: - import bz2 - except ImportError: - raise CompressionError("bz2 module is not available") - if mode == "r": - self.dbuf = "" - self.cmp = bz2.BZ2Decompressor() - else: - self.cmp = bz2.BZ2Compressor() - - def __del__(self): - if hasattr(self, "closed") and not self.closed: - self.close() - - def _init_write_gz(self): - """Initialize for writing with gzip compression. - """ - self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED, - -self.zlib.MAX_WBITS, - self.zlib.DEF_MEM_LEVEL, - 0) - timestamp = struct.pack(" self.bufsize: - self.fileobj.write(self.buf[:self.bufsize]) - self.buf = self.buf[self.bufsize:] - - def close(self): - """Close the _Stream object. No operation should be - done on it afterwards. - """ - if self.closed: - return - - if self.mode == "w" and self.comptype != "tar": - self.buf += self.cmp.flush() - - if self.mode == "w" and self.buf: - self.fileobj.write(self.buf) - self.buf = "" - if self.comptype == "gz": - # The native zlib crc is an unsigned 32-bit integer, but - # the Python wrapper implicitly casts that to a signed C - # long. So, on a 32-bit box self.crc may "look negative", - # while the same crc on a 64-bit box may "look positive". - # To avoid irksome warnings from the `struct` module, force - # it to look positive on all boxes. - self.fileobj.write(struct.pack("= 0: - blocks, remainder = divmod(pos - self.pos, self.bufsize) - for i in xrange(blocks): - self.read(self.bufsize) - self.read(remainder) - else: - raise StreamError("seeking backwards is not allowed") - return self.pos - - def read(self, size=None): - """Return the next size number of bytes from the stream. - If size is not defined, return all bytes of the stream - up to EOF. - """ - if size is None: - t = [] - while True: - buf = self._read(self.bufsize) - if not buf: - break - t.append(buf) - buf = "".join(t) - else: - buf = self._read(size) - self.pos += len(buf) - return buf - - def _read(self, size): - """Return size bytes from the stream. - """ - if self.comptype == "tar": - return self.__read(size) - - c = len(self.dbuf) - t = [self.dbuf] - while c < size: - buf = self.__read(self.bufsize) - if not buf: - break - try: - buf = self.cmp.decompress(buf) - except IOError: - raise ReadError("invalid compressed data") - t.append(buf) - c += len(buf) - t = "".join(t) - self.dbuf = t[size:] - return t[:size] - - def __read(self, size): - """Return size bytes from stream. If internal buffer is empty, - read another block from the stream. - """ - c = len(self.buf) - t = [self.buf] - while c < size: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - t.append(buf) - c += len(buf) - t = "".join(t) - self.buf = t[size:] - return t[:size] -# class _Stream - -class _StreamProxy(object): - """Small proxy class that enables transparent compression - detection for the Stream interface (mode 'r|*'). - """ - - def __init__(self, fileobj): - self.fileobj = fileobj - self.buf = self.fileobj.read(BLOCKSIZE) - - def read(self, size): - self.read = self.fileobj.read - return self.buf - - def getcomptype(self): - if self.buf.startswith("\037\213\010"): - return "gz" - if self.buf.startswith("BZh91"): - return "bz2" - return "tar" - - def close(self): - self.fileobj.close() -# class StreamProxy - -class _BZ2Proxy(object): - """Small proxy class that enables external file object - support for "r:bz2" and "w:bz2" modes. This is actually - a workaround for a limitation in bz2 module's BZ2File - class which (unlike gzip.GzipFile) has no support for - a file object argument. - """ - - blocksize = 16 * 1024 - - def __init__(self, fileobj, mode): - self.fileobj = fileobj - self.mode = mode - self.name = getattr(self.fileobj, "name", None) - self.init() - - def init(self): - import bz2 - self.pos = 0 - if self.mode == "r": - self.bz2obj = bz2.BZ2Decompressor() - self.fileobj.seek(0) - self.buf = "" - else: - self.bz2obj = bz2.BZ2Compressor() - - def read(self, size): - b = [self.buf] - x = len(self.buf) - while x < size: - raw = self.fileobj.read(self.blocksize) - if not raw: - break - data = self.bz2obj.decompress(raw) - b.append(data) - x += len(data) - self.buf = "".join(b) - - buf = self.buf[:size] - self.buf = self.buf[size:] - self.pos += len(buf) - return buf - - def seek(self, pos): - if pos < self.pos: - self.init() - self.read(pos - self.pos) - - def tell(self): - return self.pos - - def write(self, data): - self.pos += len(data) - raw = self.bz2obj.compress(data) - self.fileobj.write(raw) - - def close(self): - if self.mode == "w": - raw = self.bz2obj.flush() - self.fileobj.write(raw) -# class _BZ2Proxy - -#------------------------ -# Extraction file object -#------------------------ -class _FileInFile(object): - """A thin wrapper around an existing file object that - provides a part of its data as an individual file - object. - """ - - def __init__(self, fileobj, offset, size, sparse=None): - self.fileobj = fileobj - self.offset = offset - self.size = size - self.sparse = sparse - self.position = 0 - - def tell(self): - """Return the current file position. - """ - return self.position - - def seek(self, position): - """Seek to a position in the file. - """ - self.position = position - - def read(self, size=None): - """Read data from the file. - """ - if size is None: - size = self.size - self.position - else: - size = min(size, self.size - self.position) - - if self.sparse is None: - return self.readnormal(size) - else: - return self.readsparse(size) - - def readnormal(self, size): - """Read operation for regular files. - """ - self.fileobj.seek(self.offset + self.position) - self.position += size - return self.fileobj.read(size) - - def readsparse(self, size): - """Read operation for sparse files. - """ - data = [] - while size > 0: - buf = self.readsparsesection(size) - if not buf: - break - size -= len(buf) - data.append(buf) - return "".join(data) - - def readsparsesection(self, size): - """Read a single section of a sparse file. - """ - section = self.sparse.find(self.position) - - if section is None: - return "" - - size = min(size, section.offset + section.size - self.position) - - if isinstance(section, _data): - realpos = section.realpos + self.position - section.offset - self.fileobj.seek(self.offset + realpos) - self.position += size - return self.fileobj.read(size) - else: - self.position += size - return NUL * size -#class _FileInFile - - -class ExFileObject(object): - """File-like object for reading an archive member. - Is returned by TarFile.extractfile(). - """ - blocksize = 1024 - - def __init__(self, tarfile, tarinfo): - self.fileobj = _FileInFile(tarfile.fileobj, - tarinfo.offset_data, - tarinfo.size, - getattr(tarinfo, "sparse", None)) - self.name = tarinfo.name - self.mode = "r" - self.closed = False - self.size = tarinfo.size - - self.position = 0 - self.buffer = "" - - def read(self, size=None): - """Read at most size bytes from the file. If size is not - present or None, read all data until EOF is reached. - """ - if self.closed: - raise ValueError("I/O operation on closed file") - - buf = "" - if self.buffer: - if size is None: - buf = self.buffer - self.buffer = "" - else: - buf = self.buffer[:size] - self.buffer = self.buffer[size:] - - if size is None: - buf += self.fileobj.read() - else: - buf += self.fileobj.read(size - len(buf)) - - self.position += len(buf) - return buf - - def readline(self, size=-1): - """Read one entire line from the file. If size is present - and non-negative, return a string with at most that - size, which may be an incomplete line. - """ - if self.closed: - raise ValueError("I/O operation on closed file") - - if "\n" in self.buffer: - pos = self.buffer.find("\n") + 1 - else: - buffers = [self.buffer] - while True: - buf = self.fileobj.read(self.blocksize) - buffers.append(buf) - if not buf or "\n" in buf: - self.buffer = "".join(buffers) - pos = self.buffer.find("\n") + 1 - if pos == 0: - # no newline found. - pos = len(self.buffer) - break - - if size != -1: - pos = min(size, pos) - - buf = self.buffer[:pos] - self.buffer = self.buffer[pos:] - self.position += len(buf) - return buf - - def readlines(self): - """Return a list with all remaining lines. - """ - result = [] - while True: - line = self.readline() - if not line: break - result.append(line) - return result - - def tell(self): - """Return the current file position. - """ - if self.closed: - raise ValueError("I/O operation on closed file") - - return self.position - - def seek(self, pos, whence=os.SEEK_SET): - """Seek to a position in the file. - """ - if self.closed: - raise ValueError("I/O operation on closed file") - - if whence == os.SEEK_SET: - self.position = min(max(pos, 0), self.size) - elif whence == os.SEEK_CUR: - if pos < 0: - self.position = max(self.position + pos, 0) - else: - self.position = min(self.position + pos, self.size) - elif whence == os.SEEK_END: - self.position = max(min(self.size + pos, self.size), 0) - else: - raise ValueError("Invalid argument") - - self.buffer = "" - self.fileobj.seek(self.position) - - def close(self): - """Close the file object. - """ - self.closed = True - - def __iter__(self): - """Get an iterator over the file's lines. - """ - while True: - line = self.readline() - if not line: - break - yield line -#class ExFileObject - -#------------------ -# Exported Classes -#------------------ -class TarInfo(object): - """Informational class which holds the details about an - archive member given by a tar header block. - TarInfo objects are returned by TarFile.getmember(), - TarFile.getmembers() and TarFile.gettarinfo() and are - usually created internally. - """ - - def __init__(self, name=""): - """Construct a TarInfo object. name is the optional name - of the member. - """ - self.name = name # member name - self.mode = 0644 # file permissions - self.uid = 0 # user id - self.gid = 0 # group id - self.size = 0 # file size - self.mtime = 0 # modification time - self.chksum = 0 # header checksum - self.type = REGTYPE # member type - self.linkname = "" # link name - self.uname = "" # user name - self.gname = "" # group name - self.devmajor = 0 # device major number - self.devminor = 0 # device minor number - - self.offset = 0 # the tar header starts here - self.offset_data = 0 # the file's data starts here - - self.pax_headers = {} # pax header information - - # In pax headers the "name" and "linkname" field are called - # "path" and "linkpath". - def _getpath(self): - return self.name - def _setpath(self, name): - self.name = name - path = property(_getpath, _setpath) - - def _getlinkpath(self): - return self.linkname - def _setlinkpath(self, linkname): - self.linkname = linkname - linkpath = property(_getlinkpath, _setlinkpath) - - def __repr__(self): - return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self)) - - def get_info(self, encoding, errors): - """Return the TarInfo's attributes as a dictionary. - """ - info = { - "name": self.name, - "mode": self.mode & 07777, - "uid": self.uid, - "gid": self.gid, - "size": self.size, - "mtime": self.mtime, - "chksum": self.chksum, - "type": self.type, - "linkname": self.linkname, - "uname": self.uname, - "gname": self.gname, - "devmajor": self.devmajor, - "devminor": self.devminor - } - - if info["type"] == DIRTYPE and not info["name"].endswith("/"): - info["name"] += "/" - - for key in ("name", "linkname", "uname", "gname"): - if type(info[key]) is unicode: - info[key] = info[key].encode(encoding, errors) - - return info - - def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="strict"): - """Return a tar header as a string of 512 byte blocks. - """ - info = self.get_info(encoding, errors) - - if format == USTAR_FORMAT: - return self.create_ustar_header(info) - elif format == GNU_FORMAT: - return self.create_gnu_header(info) - elif format == PAX_FORMAT: - return self.create_pax_header(info, encoding, errors) - else: - raise ValueError("invalid format") - - def create_ustar_header(self, info): - """Return the object as a ustar header block. - """ - info["magic"] = POSIX_MAGIC - - if len(info["linkname"]) > LENGTH_LINK: - raise ValueError("linkname is too long") - - if len(info["name"]) > LENGTH_NAME: - info["prefix"], info["name"] = self._posix_split_name(info["name"]) - - return self._create_header(info, USTAR_FORMAT) - - def create_gnu_header(self, info): - """Return the object as a GNU header block sequence. - """ - info["magic"] = GNU_MAGIC - - buf = "" - if len(info["linkname"]) > LENGTH_LINK: - buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK) - - if len(info["name"]) > LENGTH_NAME: - buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME) - - return buf + self._create_header(info, GNU_FORMAT) - - def create_pax_header(self, info, encoding, errors): - """Return the object as a ustar header block. If it cannot be - represented this way, prepend a pax extended header sequence - with supplement information. - """ - info["magic"] = POSIX_MAGIC - pax_headers = self.pax_headers.copy() - - # Test string fields for values that exceed the field length or cannot - # be represented in ASCII encoding. - for name, hname, length in ( - ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK), - ("uname", "uname", 32), ("gname", "gname", 32)): - - if hname in pax_headers: - # The pax header has priority. - continue - - val = info[name].decode(encoding, errors) - - # Try to encode the string as ASCII. - try: - val.encode("ascii") - except UnicodeEncodeError: - pax_headers[hname] = val - continue - - if len(info[name]) > length: - pax_headers[hname] = val - - # Test number fields for values that exceed the field limit or values - # that like to be stored as float. - for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)): - if name in pax_headers: - # The pax header has priority. Avoid overflow. - info[name] = 0 - continue - - val = info[name] - if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float): - pax_headers[name] = unicode(val) - info[name] = 0 - - # Create a pax extended header if necessary. - if pax_headers: - buf = self._create_pax_generic_header(pax_headers) - else: - buf = "" - - return buf + self._create_header(info, USTAR_FORMAT) - - @classmethod - def create_pax_global_header(cls, pax_headers): - """Return the object as a pax global header block sequence. - """ - return cls._create_pax_generic_header(pax_headers, type=XGLTYPE) - - def _posix_split_name(self, name): - """Split a name longer than 100 chars into a prefix - and a name part. - """ - prefix = name[:LENGTH_PREFIX + 1] - while prefix and prefix[-1] != "/": - prefix = prefix[:-1] - - name = name[len(prefix):] - prefix = prefix[:-1] - - if not prefix or len(name) > LENGTH_NAME: - raise ValueError("name is too long") - return prefix, name - - @staticmethod - def _create_header(info, format): - """Return a header block. info is a dictionary with file - information, format must be one of the *_FORMAT constants. - """ - parts = [ - stn(info.get("name", ""), 100), - itn(info.get("mode", 0) & 07777, 8, format), - itn(info.get("uid", 0), 8, format), - itn(info.get("gid", 0), 8, format), - itn(info.get("size", 0), 12, format), - itn(info.get("mtime", 0), 12, format), - " ", # checksum field - info.get("type", REGTYPE), - stn(info.get("linkname", ""), 100), - stn(info.get("magic", POSIX_MAGIC), 8), - stn(info.get("uname", ""), 32), - stn(info.get("gname", ""), 32), - itn(info.get("devmajor", 0), 8, format), - itn(info.get("devminor", 0), 8, format), - stn(info.get("prefix", ""), 155) - ] - - buf = struct.pack("%ds" % BLOCKSIZE, "".join(parts)) - chksum = calc_chksums(buf[-BLOCKSIZE:])[0] - buf = buf[:-364] + "%06o\0" % chksum + buf[-357:] - return buf - - @staticmethod - def _create_payload(payload): - """Return the string payload filled with zero bytes - up to the next 512 byte border. - """ - blocks, remainder = divmod(len(payload), BLOCKSIZE) - if remainder > 0: - payload += (BLOCKSIZE - remainder) * NUL - return payload - - @classmethod - def _create_gnu_long_header(cls, name, type): - """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence - for name. - """ - name += NUL - - info = {} - info["name"] = "././@LongLink" - info["type"] = type - info["size"] = len(name) - info["magic"] = GNU_MAGIC - - # create extended header + name blocks. - return cls._create_header(info, USTAR_FORMAT) + \ - cls._create_payload(name) - - @classmethod - def _create_pax_generic_header(cls, pax_headers, type=XHDTYPE): - """Return a POSIX.1-2001 extended or global header sequence - that contains a list of keyword, value pairs. The values - must be unicode objects. - """ - records = [] - for keyword, value in pax_headers.iteritems(): - keyword = keyword.encode("utf8") - value = value.encode("utf8") - l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n' - n = p = 0 - while True: - n = l + len(str(p)) - if n == p: - break - p = n - records.append("%d %s=%s\n" % (p, keyword, value)) - records = "".join(records) - - # We use a hardcoded "././@PaxHeader" name like star does - # instead of the one that POSIX recommends. - info = {} - info["name"] = "././@PaxHeader" - info["type"] = type - info["size"] = len(records) - info["magic"] = POSIX_MAGIC - - # Create pax header + record blocks. - return cls._create_header(info, USTAR_FORMAT) + \ - cls._create_payload(records) - - @classmethod - def frombuf(cls, buf): - """Construct a TarInfo object from a 512 byte string buffer. - """ - if len(buf) == 0: - raise EmptyHeaderError("empty header") - if len(buf) != BLOCKSIZE: - raise TruncatedHeaderError("truncated header") - if buf.count(NUL) == BLOCKSIZE: - raise EOFHeaderError("end of file header") - - chksum = nti(buf[148:156]) - if chksum not in calc_chksums(buf): - raise InvalidHeaderError("bad checksum") - - obj = cls() - obj.buf = buf - obj.name = nts(buf[0:100]) - obj.mode = nti(buf[100:108]) - obj.uid = nti(buf[108:116]) - obj.gid = nti(buf[116:124]) - obj.size = nti(buf[124:136]) - obj.mtime = nti(buf[136:148]) - obj.chksum = chksum - obj.type = buf[156:157] - obj.linkname = nts(buf[157:257]) - obj.uname = nts(buf[265:297]) - obj.gname = nts(buf[297:329]) - obj.devmajor = nti(buf[329:337]) - obj.devminor = nti(buf[337:345]) - prefix = nts(buf[345:500]) - - # Old V7 tar format represents a directory as a regular - # file with a trailing slash. - if obj.type == AREGTYPE and obj.name.endswith("/"): - obj.type = DIRTYPE - - # Remove redundant slashes from directories. - if obj.isdir(): - obj.name = obj.name.rstrip("/") - - # Reconstruct a ustar longname. - if prefix and obj.type not in GNU_TYPES: - obj.name = prefix + "/" + obj.name - return obj - - @classmethod - def fromtarfile(cls, tarfile): - """Return the next TarInfo object from TarFile object - tarfile. - """ - buf = tarfile.fileobj.read(BLOCKSIZE) - obj = cls.frombuf(buf) - obj.offset = tarfile.fileobj.tell() - BLOCKSIZE - return obj._proc_member(tarfile) - - #-------------------------------------------------------------------------- - # The following are methods that are called depending on the type of a - # member. The entry point is _proc_member() which can be overridden in a - # subclass to add custom _proc_*() methods. A _proc_*() method MUST - # implement the following - # operations: - # 1. Set self.offset_data to the position where the data blocks begin, - # if there is data that follows. - # 2. Set tarfile.offset to the position where the next member's header will - # begin. - # 3. Return self or another valid TarInfo object. - def _proc_member(self, tarfile): - """Choose the right processing method depending on - the type and call it. - """ - if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK): - return self._proc_gnulong(tarfile) - elif self.type == GNUTYPE_SPARSE: - return self._proc_sparse(tarfile) - elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE): - return self._proc_pax(tarfile) - else: - return self._proc_builtin(tarfile) - - def _proc_builtin(self, tarfile): - """Process a builtin type or an unknown type which - will be treated as a regular file. - """ - self.offset_data = tarfile.fileobj.tell() - offset = self.offset_data - if self.isreg() or self.type not in SUPPORTED_TYPES: - # Skip the following data blocks. - offset += self._block(self.size) - tarfile.offset = offset - - # Patch the TarInfo object with saved global - # header information. - self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors) - - return self - - def _proc_gnulong(self, tarfile): - """Process the blocks that hold a GNU longname - or longlink member. - """ - buf = tarfile.fileobj.read(self._block(self.size)) - - # Fetch the next header and process it. - try: - next = self.fromtarfile(tarfile) - except HeaderError: - raise SubsequentHeaderError("missing or bad subsequent header") - - # Patch the TarInfo object from the next header with - # the longname information. - next.offset = self.offset - if self.type == GNUTYPE_LONGNAME: - next.name = nts(buf) - elif self.type == GNUTYPE_LONGLINK: - next.linkname = nts(buf) - - return next - - def _proc_sparse(self, tarfile): - """Process a GNU sparse header plus extra headers. - """ - buf = self.buf - sp = _ringbuffer() - pos = 386 - lastpos = 0L - realpos = 0L - # There are 4 possible sparse structs in the - # first header. - for i in xrange(4): - try: - offset = nti(buf[pos:pos + 12]) - numbytes = nti(buf[pos + 12:pos + 24]) - except ValueError: - break - if offset > lastpos: - sp.append(_hole(lastpos, offset - lastpos)) - sp.append(_data(offset, numbytes, realpos)) - realpos += numbytes - lastpos = offset + numbytes - pos += 24 - - isextended = ord(buf[482]) - origsize = nti(buf[483:495]) - - # If the isextended flag is given, - # there are extra headers to process. - while isextended == 1: - buf = tarfile.fileobj.read(BLOCKSIZE) - pos = 0 - for i in xrange(21): - try: - offset = nti(buf[pos:pos + 12]) - numbytes = nti(buf[pos + 12:pos + 24]) - except ValueError: - break - if offset > lastpos: - sp.append(_hole(lastpos, offset - lastpos)) - sp.append(_data(offset, numbytes, realpos)) - realpos += numbytes - lastpos = offset + numbytes - pos += 24 - isextended = ord(buf[504]) - - if lastpos < origsize: - sp.append(_hole(lastpos, origsize - lastpos)) - - self.sparse = sp - - self.offset_data = tarfile.fileobj.tell() - tarfile.offset = self.offset_data + self._block(self.size) - self.size = origsize - - return self - - def _proc_pax(self, tarfile): - """Process an extended or global header as described in - POSIX.1-2001. - """ - # Read the header information. - buf = tarfile.fileobj.read(self._block(self.size)) - - # A pax header stores supplemental information for either - # the following file (extended) or all following files - # (global). - if self.type == XGLTYPE: - pax_headers = tarfile.pax_headers - else: - pax_headers = tarfile.pax_headers.copy() - - # Parse pax header information. A record looks like that: - # "%d %s=%s\n" % (length, keyword, value). length is the size - # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(r"(\d+) ([^=]+)=", re.U) - pos = 0 - while True: - match = regex.match(buf, pos) - if not match: - break - - length, keyword = match.groups() - length = int(length) - value = buf[match.end(2) + 1:match.start(1) + length - 1] - - keyword = keyword.decode("utf8") - value = value.decode("utf8") - - pax_headers[keyword] = value - pos += length - - # Fetch the next header. - try: - next = self.fromtarfile(tarfile) - except HeaderError: - raise SubsequentHeaderError("missing or bad subsequent header") - - if self.type in (XHDTYPE, SOLARIS_XHDTYPE): - # Patch the TarInfo object with the extended header info. - next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) - next.offset = self.offset - - if "size" in pax_headers: - # If the extended header replaces the size field, - # we need to recalculate the offset where the next - # header starts. - offset = next.offset_data - if next.isreg() or next.type not in SUPPORTED_TYPES: - offset += next._block(next.size) - tarfile.offset = offset - - return next - - def _apply_pax_info(self, pax_headers, encoding, errors): - """Replace fields with supplemental information from a previous - pax extended or global header. - """ - for keyword, value in pax_headers.iteritems(): - if keyword not in PAX_FIELDS: - continue - - if keyword == "path": - value = value.rstrip("/") - - if keyword in PAX_NUMBER_FIELDS: - try: - value = PAX_NUMBER_FIELDS[keyword](value) - except ValueError: - value = 0 - else: - value = uts(value, encoding, errors) - - setattr(self, keyword, value) - - self.pax_headers = pax_headers.copy() - - def _block(self, count): - """Round up a byte count by BLOCKSIZE and return it, - e.g. _block(834) => 1024. - """ - blocks, remainder = divmod(count, BLOCKSIZE) - if remainder: - blocks += 1 - return blocks * BLOCKSIZE - - def isreg(self): - return self.type in REGULAR_TYPES - def isfile(self): - return self.isreg() - def isdir(self): - return self.type == DIRTYPE - def issym(self): - return self.type == SYMTYPE - def islnk(self): - return self.type == LNKTYPE - def ischr(self): - return self.type == CHRTYPE - def isblk(self): - return self.type == BLKTYPE - def isfifo(self): - return self.type == FIFOTYPE - def issparse(self): - return self.type == GNUTYPE_SPARSE - def isdev(self): - return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE) -# class TarInfo - -class TarFile(object): - """The TarFile Class provides an interface to tar archives. - """ - - debug = 0 # May be set from 0 (no msgs) to 3 (all msgs) - - dereference = False # If true, add content of linked file to the - # tar file, else the link. - - ignore_zeros = False # If true, skips empty or invalid blocks and - # continues processing. - - errorlevel = 1 # If 0, fatal errors only appear in debug - # messages (if debug >= 0). If > 0, errors - # are passed to the caller as exceptions. - - format = DEFAULT_FORMAT # The format to use when creating an archive. - - encoding = ENCODING # Encoding for 8-bit character strings. - - errors = None # Error handler for unicode conversion. - - tarinfo = TarInfo # The default TarInfo class to use. - - fileobject = ExFileObject # The default ExFileObject class to use. - - def __init__(self, name=None, mode="r", fileobj=None, format=None, - tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, - errors=None, pax_headers=None, debug=None, errorlevel=None): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to - read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' - defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. - """ - if len(mode) > 1 or mode not in "raw": - raise ValueError("mode must be 'r', 'a' or 'w'") - self.mode = mode - self._mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode] - - if not fileobj: - if self.mode == "a" and not os.path.exists(name): - # Create nonexistent files in append mode. - self.mode = "w" - self._mode = "wb" - fileobj = bltn_open(name, self._mode) - self._extfileobj = False - else: - if name is None and hasattr(fileobj, "name"): - name = fileobj.name - if hasattr(fileobj, "mode"): - self._mode = fileobj.mode - self._extfileobj = True - self.name = os.path.abspath(name) if name else None - self.fileobj = fileobj - - # Init attributes. - if format is not None: - self.format = format - if tarinfo is not None: - self.tarinfo = tarinfo - if dereference is not None: - self.dereference = dereference - if ignore_zeros is not None: - self.ignore_zeros = ignore_zeros - if encoding is not None: - self.encoding = encoding - - if errors is not None: - self.errors = errors - elif mode == "r": - self.errors = "utf-8" - else: - self.errors = "strict" - - if pax_headers is not None and self.format == PAX_FORMAT: - self.pax_headers = pax_headers - else: - self.pax_headers = {} - - if debug is not None: - self.debug = debug - if errorlevel is not None: - self.errorlevel = errorlevel - - # Init datastructures. - self.closed = False - self.members = [] # list of members as TarInfo objects - self._loaded = False # flag if all members have been read - self.offset = self.fileobj.tell() - # current position in the archive file - self.inodes = {} # dictionary caching the inodes of - # archive members already added - - try: - if self.mode == "r": - self.firstmember = None - self.firstmember = self.next() - - if self.mode == "a": - # Move to the end of the archive, - # before the first empty block. - while True: - self.fileobj.seek(self.offset) - try: - tarinfo = self.tarinfo.fromtarfile(self) - self.members.append(tarinfo) - except EOFHeaderError: - self.fileobj.seek(self.offset) - break - except HeaderError, e: - raise ReadError(str(e)) - - if self.mode in "aw": - self._loaded = True - - if self.pax_headers: - buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy()) - self.fileobj.write(buf) - self.offset += len(buf) - except: - if not self._extfileobj: - self.fileobj.close() - self.closed = True - raise - - def _getposix(self): - return self.format == USTAR_FORMAT - def _setposix(self, value): - import warnings - warnings.warn("use the format attribute instead", DeprecationWarning, - 2) - if value: - self.format = USTAR_FORMAT - else: - self.format = GNU_FORMAT - posix = property(_getposix, _setposix) - - #-------------------------------------------------------------------------- - # Below are the classmethods which act as alternate constructors to the - # TarFile class. The open() method is the only one that is needed for - # public use; it is the "super"-constructor and is able to select an - # adequate "sub"-constructor for a particular compression using the mapping - # from OPEN_METH. - # - # This concept allows one to subclass TarFile without losing the comfort of - # the super-constructor. A sub-constructor is registered and made available - # by adding it to the mapping in OPEN_METH. - - @classmethod - def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): - """Open a tar archive for reading, writing or appending. Return - an appropriate TarFile class. - - mode: - 'r' or 'r:*' open for reading with transparent compression - 'r:' open for reading exclusively uncompressed - 'r:gz' open for reading with gzip compression - 'r:bz2' open for reading with bzip2 compression - 'a' or 'a:' open for appending, creating the file if necessary - 'w' or 'w:' open for writing without compression - 'w:gz' open for writing with gzip compression - 'w:bz2' open for writing with bzip2 compression - - 'r|*' open a stream of tar blocks with transparent compression - 'r|' open an uncompressed stream of tar blocks for reading - 'r|gz' open a gzip compressed stream of tar blocks - 'r|bz2' open a bzip2 compressed stream of tar blocks - 'w|' open an uncompressed stream for writing - 'w|gz' open a gzip compressed stream for writing - 'w|bz2' open a bzip2 compressed stream for writing - """ - - if not name and not fileobj: - raise ValueError("nothing to open") - - if mode in ("r", "r:*"): - # Find out which *open() is appropriate for opening the file. - for comptype in cls.OPEN_METH: - func = getattr(cls, cls.OPEN_METH[comptype]) - if fileobj is not None: - saved_pos = fileobj.tell() - try: - return func(name, "r", fileobj, **kwargs) - except (ReadError, CompressionError), e: - if fileobj is not None: - fileobj.seek(saved_pos) - continue - raise ReadError("file could not be opened successfully") - - elif ":" in mode: - filemode, comptype = mode.split(":", 1) - filemode = filemode or "r" - comptype = comptype or "tar" - - # Select the *open() function according to - # given compression. - if comptype in cls.OPEN_METH: - func = getattr(cls, cls.OPEN_METH[comptype]) - else: - raise CompressionError("unknown compression type %r" % comptype) - return func(name, filemode, fileobj, **kwargs) - - elif "|" in mode: - filemode, comptype = mode.split("|", 1) - filemode = filemode or "r" - comptype = comptype or "tar" - - if filemode not in "rw": - raise ValueError("mode must be 'r' or 'w'") - - t = cls(name, filemode, - _Stream(name, filemode, comptype, fileobj, bufsize), - **kwargs) - t._extfileobj = False - return t - - elif mode in "aw": - return cls.taropen(name, mode, fileobj, **kwargs) - - raise ValueError("undiscernible mode") - - @classmethod - def taropen(cls, name, mode="r", fileobj=None, **kwargs): - """Open uncompressed tar archive name for reading or writing. - """ - if len(mode) > 1 or mode not in "raw": - raise ValueError("mode must be 'r', 'a' or 'w'") - return cls(name, mode, fileobj, **kwargs) - - @classmethod - def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): - """Open gzip compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if len(mode) > 1 or mode not in "rw": - raise ValueError("mode must be 'r' or 'w'") - - try: - import gzip - gzip.GzipFile - except (ImportError, AttributeError): - raise CompressionError("gzip module is not available") - - if fileobj is None: - fileobj = bltn_open(name, mode + "b") - - try: - t = cls.taropen(name, mode, - gzip.GzipFile(name, mode, compresslevel, fileobj), - **kwargs) - except IOError: - raise ReadError("not a gzip file") - t._extfileobj = False - return t - - @classmethod - def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): - """Open bzip2 compressed tar archive name for reading or writing. - Appending is not allowed. - """ - if len(mode) > 1 or mode not in "rw": - raise ValueError("mode must be 'r' or 'w'.") - - try: - import bz2 - except ImportError: - raise CompressionError("bz2 module is not available") - - if fileobj is not None: - fileobj = _BZ2Proxy(fileobj, mode) - else: - fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel) - - try: - t = cls.taropen(name, mode, fileobj, **kwargs) - except (IOError, EOFError): - raise ReadError("not a bzip2 file") - t._extfileobj = False - return t - - # All *open() methods are registered here. - OPEN_METH = { - "tar": "taropen", # uncompressed tar - "gz": "gzopen", # gzip compressed tar - "bz2": "bz2open" # bzip2 compressed tar - } - - #-------------------------------------------------------------------------- - # The public methods which TarFile provides: - - def close(self): - """Close the TarFile. In write-mode, two finishing zero blocks are - appended to the archive. - """ - if self.closed: - return - - if self.mode in "aw": - self.fileobj.write(NUL * (BLOCKSIZE * 2)) - self.offset += (BLOCKSIZE * 2) - # fill up the end with zero-blocks - # (like option -b20 for tar does) - blocks, remainder = divmod(self.offset, RECORDSIZE) - if remainder > 0: - self.fileobj.write(NUL * (RECORDSIZE - remainder)) - - if not self._extfileobj: - self.fileobj.close() - self.closed = True - - def getmember(self, name): - """Return a TarInfo object for member `name'. If `name' can not be - found in the archive, KeyError is raised. If a member occurs more - than once in the archive, its last occurrence is assumed to be the - most up-to-date version. - """ - tarinfo = self._getmember(name) - if tarinfo is None: - raise KeyError("filename %r not found" % name) - return tarinfo - - def getmembers(self): - """Return the members of the archive as a list of TarInfo objects. The - list has the same order as the members in the archive. - """ - self._check() - if not self._loaded: # if we want to obtain a list of - self._load() # all members, we first have to - # scan the whole archive. - return self.members - - def getnames(self): - """Return the members of the archive as a list of their names. It has - the same order as the list returned by getmembers(). - """ - return [tarinfo.name for tarinfo in self.getmembers()] - - def gettarinfo(self, name=None, arcname=None, fileobj=None): - """Create a TarInfo object for either the file `name' or the file - object `fileobj' (using os.fstat on its file descriptor). You can - modify some of the TarInfo's attributes before you add it using - addfile(). If given, `arcname' specifies an alternative name for the - file in the archive. - """ - self._check("aw") - - # When fileobj is given, replace name by - # fileobj's real name. - if fileobj is not None: - name = fileobj.name - - # Building the name of the member in the archive. - # Backward slashes are converted to forward slashes, - # Absolute paths are turned to relative paths. - if arcname is None: - arcname = name - drv, arcname = os.path.splitdrive(arcname) - arcname = arcname.replace(os.sep, "/") - arcname = arcname.lstrip("/") - - # Now, fill the TarInfo object with - # information specific for the file. - tarinfo = self.tarinfo() - tarinfo.tarfile = self - - # Use os.stat or os.lstat, depending on platform - # and if symlinks shall be resolved. - if fileobj is None: - if hasattr(os, "lstat") and not self.dereference: - statres = os.lstat(name) - else: - statres = os.stat(name) - else: - statres = os.fstat(fileobj.fileno()) - linkname = "" - - stmd = statres.st_mode - if stat.S_ISREG(stmd): - inode = (statres.st_ino, statres.st_dev) - if not self.dereference and statres.st_nlink > 1 and \ - inode in self.inodes and arcname != self.inodes[inode]: - # Is it a hardlink to an already - # archived file? - type = LNKTYPE - linkname = self.inodes[inode] - else: - # The inode is added only if its valid. - # For win32 it is always 0. - type = REGTYPE - if inode[0]: - self.inodes[inode] = arcname - elif stat.S_ISDIR(stmd): - type = DIRTYPE - elif stat.S_ISFIFO(stmd): - type = FIFOTYPE - elif stat.S_ISLNK(stmd): - type = SYMTYPE - linkname = os.readlink(name) - elif stat.S_ISCHR(stmd): - type = CHRTYPE - elif stat.S_ISBLK(stmd): - type = BLKTYPE - else: - return None - - # Fill the TarInfo object with all - # information we can get. - tarinfo.name = arcname - tarinfo.mode = stmd - tarinfo.uid = statres.st_uid - tarinfo.gid = statres.st_gid - if type == REGTYPE: - tarinfo.size = statres.st_size - else: - tarinfo.size = 0L - tarinfo.mtime = statres.st_mtime - tarinfo.type = type - tarinfo.linkname = linkname - if pwd: - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass - if grp: - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass - - if type in (CHRTYPE, BLKTYPE): - if hasattr(os, "major") and hasattr(os, "minor"): - tarinfo.devmajor = os.major(statres.st_rdev) - tarinfo.devminor = os.minor(statres.st_rdev) - return tarinfo - - def list(self, verbose=True): - """Print a table of contents to sys.stdout. If `verbose' is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. - """ - self._check() - - for tarinfo in self: - if verbose: - print filemode(tarinfo.mode), - print "%s/%s" % (tarinfo.uname or tarinfo.uid, - tarinfo.gname or tarinfo.gid), - if tarinfo.ischr() or tarinfo.isblk(): - print "%10s" % ("%d,%d" \ - % (tarinfo.devmajor, tarinfo.devminor)), - else: - print "%10d" % tarinfo.size, - print "%d-%02d-%02d %02d:%02d:%02d" \ - % time.localtime(tarinfo.mtime)[:6], - - print tarinfo.name + ("/" if tarinfo.isdir() else ""), - - if verbose: - if tarinfo.issym(): - print "->", tarinfo.linkname, - if tarinfo.islnk(): - print "link to", tarinfo.linkname, - print - - def add(self, name, arcname=None, recursive=True, exclude=None, filter=None): - """Add the file `name' to the archive. `name' may be any type of file - (directory, fifo, symbolic link, etc.). If given, `arcname' - specifies an alternative name for the file in the archive. - Directories are added recursively by default. This can be avoided by - setting `recursive' to False. `exclude' is a function that should - return True for each filename to be excluded. `filter' is a function - that expects a TarInfo object argument and returns the changed - TarInfo object, if it returns None the TarInfo object will be - excluded from the archive. - """ - self._check("aw") - - if arcname is None: - arcname = name - - # Exclude pathnames. - if exclude is not None: - import warnings - warnings.warn("use the filter argument instead", - DeprecationWarning, 2) - if exclude(name): - self._dbg(2, "tarfile: Excluded %r" % name) - return - - # Skip if somebody tries to archive the archive... - if self.name is not None and os.path.abspath(name) == self.name: - self._dbg(2, "tarfile: Skipped %r" % name) - return - - self._dbg(1, name) - - # Create a TarInfo object from the file. - tarinfo = self.gettarinfo(name, arcname) - - if tarinfo is None: - self._dbg(1, "tarfile: Unsupported type %r" % name) - return - - # Change or exclude the TarInfo object. - if filter is not None: - tarinfo = filter(tarinfo) - if tarinfo is None: - self._dbg(2, "tarfile: Excluded %r" % name) - return - - # Append the tar header and data to the archive. - if tarinfo.isreg(): - f = bltn_open(name, "rb") - self.addfile(tarinfo, f) - f.close() - - elif tarinfo.isdir(): - self.addfile(tarinfo) - if recursive: - for f in os.listdir(name): - self.add(os.path.join(name, f), os.path.join(arcname, f), - recursive, exclude, filter) - - else: - self.addfile(tarinfo) - - def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is - given, tarinfo.size bytes are read from it and added to the archive. - You can create TarInfo objects using gettarinfo(). - On Windows platforms, `fileobj' should always be opened with mode - 'rb' to avoid irritation about the file size. - """ - self._check("aw") - - tarinfo = copy.copy(tarinfo) - - buf = tarinfo.tobuf(self.format, self.encoding, self.errors) - self.fileobj.write(buf) - self.offset += len(buf) - - # If there's data to follow, append it. - if fileobj is not None: - copyfileobj(fileobj, self.fileobj, tarinfo.size) - blocks, remainder = divmod(tarinfo.size, BLOCKSIZE) - if remainder > 0: - self.fileobj.write(NUL * (BLOCKSIZE - remainder)) - blocks += 1 - self.offset += blocks * BLOCKSIZE - - self.members.append(tarinfo) - - def extractall(self, path=".", members=None): - """Extract all members from the archive to the current working - directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). - """ - directories = [] - - if members is None: - members = self - - for tarinfo in members: - if tarinfo.isdir(): - # Extract directories with a safe mode. - directories.append(tarinfo) - tarinfo = copy.copy(tarinfo) - tarinfo.mode = 0700 - self.extract(tarinfo, path) - - # Reverse sort directories. - directories.sort(key=operator.attrgetter('name')) - directories.reverse() - - # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) - try: - self.chown(tarinfo, dirpath) - self.utime(tarinfo, dirpath) - self.chmod(tarinfo, dirpath) - except ExtractError, e: - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) - - def extract(self, member, path=""): - """Extract a member from the archive to the current working directory, - using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. - """ - self._check("r") - - if isinstance(member, basestring): - tarinfo = self.getmember(member) - else: - tarinfo = member - - # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo._link_target = os.path.join(path, tarinfo.linkname) - - try: - self._extract_member(tarinfo, os.path.join(path, tarinfo.name)) - except EnvironmentError, e: - if self.errorlevel > 0: - raise - else: - if e.filename is None: - self._dbg(1, "tarfile: %s" % e.strerror) - else: - self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) - except ExtractError, e: - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) - - def extractfile(self, member): - """Extract a member from the archive as a file object. `member' may be - a filename or a TarInfo object. If `member' is a regular file, a - file-like object is returned. If `member' is a link, a file-like - object is constructed from the link's target. If `member' is none of - the above, None is returned. - The file-like object is read-only and provides the following - methods: read(), readline(), readlines(), seek() and tell() - """ - self._check("r") - - if isinstance(member, basestring): - tarinfo = self.getmember(member) - else: - tarinfo = member - - if tarinfo.isreg(): - return self.fileobject(self, tarinfo) - - elif tarinfo.type not in SUPPORTED_TYPES: - # If a member's type is unknown, it is treated as a - # regular file. - return self.fileobject(self, tarinfo) - - elif tarinfo.islnk() or tarinfo.issym(): - if isinstance(self.fileobj, _Stream): - # A small but ugly workaround for the case that someone tries - # to extract a (sym)link as a file-object from a non-seekable - # stream of tar blocks. - raise StreamError("cannot extract (sym)link as file object") - else: - # A (sym)link's file object is its target's file object. - return self.extractfile(self._find_link_target(tarinfo)) - else: - # If there's no data associated with the member (directory, chrdev, - # blkdev, etc.), return None instead of a file object. - return None - - def _extract_member(self, tarinfo, targetpath): - """Extract the TarInfo object tarinfo to a physical - file called targetpath. - """ - # Fetch the TarInfo object for the given name - # and build the destination pathname, replacing - # forward slashes to platform specific separators. - targetpath = targetpath.rstrip("/") - targetpath = targetpath.replace("/", os.sep) - - # Create all upper directories. - upperdirs = os.path.dirname(targetpath) - if upperdirs and not os.path.exists(upperdirs): - # Create directories that are not part of the archive with - # default permissions. - os.makedirs(upperdirs) - - if tarinfo.islnk() or tarinfo.issym(): - self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) - else: - self._dbg(1, tarinfo.name) - - if tarinfo.isreg(): - self.makefile(tarinfo, targetpath) - elif tarinfo.isdir(): - self.makedir(tarinfo, targetpath) - elif tarinfo.isfifo(): - self.makefifo(tarinfo, targetpath) - elif tarinfo.ischr() or tarinfo.isblk(): - self.makedev(tarinfo, targetpath) - elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) - elif tarinfo.type not in SUPPORTED_TYPES: - self.makeunknown(tarinfo, targetpath) - else: - self.makefile(tarinfo, targetpath) - - self.chown(tarinfo, targetpath) - if not tarinfo.issym(): - self.chmod(tarinfo, targetpath) - self.utime(tarinfo, targetpath) - - #-------------------------------------------------------------------------- - # Below are the different file methods. They are called via - # _extract_member() when extract() is called. They can be replaced in a - # subclass to implement other functionality. - - def makedir(self, tarinfo, targetpath): - """Make a directory called targetpath. - """ - try: - # Use a safe mode for the directory, the real mode is set - # later in _extract_member(). - os.mkdir(targetpath, 0700) - except EnvironmentError, e: - if e.errno != errno.EEXIST: - raise - - def makefile(self, tarinfo, targetpath): - """Make a file called targetpath. - """ - source = self.extractfile(tarinfo) - target = bltn_open(targetpath, "wb") - copyfileobj(source, target) - source.close() - target.close() - - def makeunknown(self, tarinfo, targetpath): - """Make a file from a TarInfo object with an unknown type - at targetpath. - """ - self.makefile(tarinfo, targetpath) - self._dbg(1, "tarfile: Unknown file type %r, " \ - "extracted as regular file." % tarinfo.type) - - def makefifo(self, tarinfo, targetpath): - """Make a fifo called targetpath. - """ - if hasattr(os, "mkfifo"): - os.mkfifo(targetpath) - else: - raise ExtractError("fifo not supported by system") - - def makedev(self, tarinfo, targetpath): - """Make a character or block device called targetpath. - """ - if not hasattr(os, "mknod") or not hasattr(os, "makedev"): - raise ExtractError("special devices not supported by system") - - mode = tarinfo.mode - if tarinfo.isblk(): - mode |= stat.S_IFBLK - else: - mode |= stat.S_IFCHR - - os.mknod(targetpath, mode, - os.makedev(tarinfo.devmajor, tarinfo.devminor)) - - def makelink(self, tarinfo, targetpath): - """Make a (symbolic) link called targetpath. If it cannot be created - (platform limitation), we try to make a copy of the referenced file - instead of a link. - """ - if hasattr(os, "symlink") and hasattr(os, "link"): - # For systems that support symbolic and hard links. - if tarinfo.issym(): - if os.path.lexists(targetpath): - os.unlink(targetpath) - os.symlink(tarinfo.linkname, targetpath) - else: - # See extract(). - if os.path.exists(tarinfo._link_target): - if os.path.lexists(targetpath): - os.unlink(targetpath) - os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), targetpath) - else: - try: - self._extract_member(self._find_link_target(tarinfo), targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") - - def chown(self, tarinfo, targetpath): - """Set owner of targetpath according to tarinfo. - """ - if pwd and hasattr(os, "geteuid") and os.geteuid() == 0: - # We have to be root to do so. - try: - g = grp.getgrnam(tarinfo.gname)[2] - except KeyError: - try: - g = grp.getgrgid(tarinfo.gid)[2] - except KeyError: - g = os.getgid() - try: - u = pwd.getpwnam(tarinfo.uname)[2] - except KeyError: - try: - u = pwd.getpwuid(tarinfo.uid)[2] - except KeyError: - u = os.getuid() - try: - if tarinfo.issym() and hasattr(os, "lchown"): - os.lchown(targetpath, u, g) - else: - if sys.platform != "os2emx": - os.chown(targetpath, u, g) - except EnvironmentError, e: - raise ExtractError("could not change owner") - - def chmod(self, tarinfo, targetpath): - """Set file permissions of targetpath according to tarinfo. - """ - if hasattr(os, 'chmod'): - try: - os.chmod(targetpath, tarinfo.mode) - except EnvironmentError, e: - raise ExtractError("could not change mode") - - def utime(self, tarinfo, targetpath): - """Set modification time of targetpath according to tarinfo. - """ - if not hasattr(os, 'utime'): - return - try: - os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime)) - except EnvironmentError, e: - raise ExtractError("could not change modification time") - - #-------------------------------------------------------------------------- - def next(self): - """Return the next member of the archive as a TarInfo object, when - TarFile is opened for reading. Return None if there is no more - available. - """ - self._check("ra") - if self.firstmember is not None: - m = self.firstmember - self.firstmember = None - return m - - # Read the next block. - self.fileobj.seek(self.offset) - tarinfo = None - while True: - try: - tarinfo = self.tarinfo.fromtarfile(self) - except EOFHeaderError, e: - if self.ignore_zeros: - self._dbg(2, "0x%X: %s" % (self.offset, e)) - self.offset += BLOCKSIZE - continue - except InvalidHeaderError, e: - if self.ignore_zeros: - self._dbg(2, "0x%X: %s" % (self.offset, e)) - self.offset += BLOCKSIZE - continue - elif self.offset == 0: - raise ReadError(str(e)) - except EmptyHeaderError: - if self.offset == 0: - raise ReadError("empty file") - except TruncatedHeaderError, e: - if self.offset == 0: - raise ReadError(str(e)) - except SubsequentHeaderError, e: - raise ReadError(str(e)) - break - - if tarinfo is not None: - self.members.append(tarinfo) - else: - self._loaded = True - - return tarinfo - - #-------------------------------------------------------------------------- - # Little helper methods: - - def _getmember(self, name, tarinfo=None, normalize=False): - """Find an archive member by name from bottom to top. - If tarinfo is given, it is used as the starting point. - """ - # Ensure that all members have been loaded. - members = self.getmembers() - - # Limit the member search list up to tarinfo. - if tarinfo is not None: - members = members[:members.index(tarinfo)] - - if normalize: - name = os.path.normpath(name) - - for member in reversed(members): - if normalize: - member_name = os.path.normpath(member.name) - else: - member_name = member.name - - if name == member_name: - return member - - def _load(self): - """Read through the entire archive file and look for readable - members. - """ - while True: - tarinfo = self.next() - if tarinfo is None: - break - self._loaded = True - - def _check(self, mode=None): - """Check if TarFile is still open, and if the operation's mode - corresponds to TarFile's mode. - """ - if self.closed: - raise IOError("%s is closed" % self.__class__.__name__) - if mode is not None and self.mode not in mode: - raise IOError("bad operation for mode %r" % self.mode) - - def _find_link_target(self, tarinfo): - """Find the target member of a symlink or hardlink member in the - archive. - """ - if tarinfo.issym(): - # Always search the entire archive. - linkname = os.path.dirname(tarinfo.name) + "/" + tarinfo.linkname - limit = None - else: - # Search the archive before the link, because a hard link is - # just a reference to an already archived file. - linkname = tarinfo.linkname - limit = tarinfo - - member = self._getmember(linkname, tarinfo=limit, normalize=True) - if member is None: - raise KeyError("linkname %r not found" % linkname) - return member - - def __iter__(self): - """Provide an iterator object. - """ - if self._loaded: - return iter(self.members) - else: - return TarIter(self) - - def _dbg(self, level, msg): - """Write debugging output to sys.stderr. - """ - if level <= self.debug: - print >> sys.stderr, msg - - def __enter__(self): - self._check() - return self - - def __exit__(self, type, value, traceback): - if type is None: - self.close() - else: - # An exception occurred. We must not call close() because - # it would try to write end-of-archive blocks and padding. - if not self._extfileobj: - self.fileobj.close() - self.closed = True -# class TarFile - -class TarIter: - """Iterator Class. - - for tarinfo in TarFile(...): - suite... - """ - - def __init__(self, tarfile): - """Construct a TarIter object. - """ - self.tarfile = tarfile - self.index = 0 - def __iter__(self): - """Return iterator object. - """ - return self - def next(self): - """Return the next item using TarFile's next() method. - When all members have been read, set TarFile as _loaded. - """ - # Fix for SF #1100429: Under rare circumstances it can - # happen that getmembers() is called during iteration, - # which will cause TarIter to stop prematurely. - if not self.tarfile._loaded: - tarinfo = self.tarfile.next() - if not tarinfo: - self.tarfile._loaded = True - raise StopIteration - else: - try: - tarinfo = self.tarfile.members[self.index] - except IndexError: - raise StopIteration - self.index += 1 - return tarinfo - -# Helper classes for sparse file support -class _section: - """Base class for _data and _hole. - """ - def __init__(self, offset, size): - self.offset = offset - self.size = size - def __contains__(self, offset): - return self.offset <= offset < self.offset + self.size - -class _data(_section): - """Represent a data section in a sparse file. - """ - def __init__(self, offset, size, realpos): - _section.__init__(self, offset, size) - self.realpos = realpos - -class _hole(_section): - """Represent a hole section in a sparse file. - """ - pass - -class _ringbuffer(list): - """Ringbuffer class which increases performance - over a regular list. - """ - def __init__(self): - self.idx = 0 - def find(self, offset): - idx = self.idx - while True: - item = self[idx] - if offset in item: - break - idx += 1 - if idx == len(self): - idx = 0 - if idx == self.idx: - # End of File - return None - self.idx = idx - return item - -#--------------------------------------------- -# zipfile compatible TarFile class -#--------------------------------------------- -TAR_PLAIN = 0 # zipfile.ZIP_STORED -TAR_GZIPPED = 8 # zipfile.ZIP_DEFLATED -class TarFileCompat: - """TarFile class compatible with standard module zipfile's - ZipFile class. - """ - def __init__(self, file, mode="r", compression=TAR_PLAIN): - from warnings import warnpy3k - warnpy3k("the TarFileCompat class has been removed in Python 3.0", - stacklevel=2) - if compression == TAR_PLAIN: - self.tarfile = TarFile.taropen(file, mode) - elif compression == TAR_GZIPPED: - self.tarfile = TarFile.gzopen(file, mode) - else: - raise ValueError("unknown compression constant") - if mode[0:1] == "r": - members = self.tarfile.getmembers() - for m in members: - m.filename = m.name - m.file_size = m.size - m.date_time = time.gmtime(m.mtime)[:6] - def namelist(self): - return map(lambda m: m.name, self.infolist()) - def infolist(self): - return filter(lambda m: m.type in REGULAR_TYPES, - self.tarfile.getmembers()) - def printdir(self): - self.tarfile.list() - def testzip(self): - return - def getinfo(self, name): - return self.tarfile.getmember(name) - def read(self, name): - return self.tarfile.extractfile(self.tarfile.getmember(name)).read() - def write(self, filename, arcname=None, compress_type=None): - self.tarfile.add(filename, arcname) - def writestr(self, zinfo, bytes): - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - import calendar - tinfo = TarInfo(zinfo.filename) - tinfo.size = len(bytes) - tinfo.mtime = calendar.timegm(zinfo.date_time) - self.tarfile.addfile(tinfo, StringIO(bytes)) - def close(self): - self.tarfile.close() -#class TarFileCompat - -#-------------------- -# exported functions -#-------------------- -def is_tarfile(name): - """Return True if name points to a tar archive that we - are able to handle, else return False. - """ - try: - t = open(name) - t.close() - return True - except TarError: - return False - -bltn_open = open -open = TarFile.open diff --git a/mat/__init__.py b/mat/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/mat/__init__.py @@ -0,0 +1 @@ + diff --git a/mat/archive.py b/mat/archive.py new file mode 100644 index 0000000..77db71c --- /dev/null +++ b/mat/archive.py @@ -0,0 +1,289 @@ +''' + Take care of archives formats +''' + +import zipfile +import shutil +import os +import logging +import tempfile + +import parser +import mat +from tarfile import tarfile + + +class GenericArchiveStripper(parser.GenericParser): + ''' + Represent a generic archive + ''' + def __init__(self, filename, parser, mime, backup, add2archive): + super(GenericArchiveStripper, self).__init__(filename, parser, mime, + backup, add2archive) + self.compression = '' + self.add2archive = add2archive + self.tempdir = tempfile.mkdtemp() + + def __del__(self): + ''' + Remove the files inside the temp dir, + then remove the temp dir + ''' + for root, dirs, files in os.walk(self.tempdir): + for item in files: + path_file = os.path.join(root, item) + mat.secure_remove(path_file) + shutil.rmtree(self.tempdir) + + def remove_all(self): + ''' + Call _remove_all() with in argument : "normal" + ''' + self._remove_all('normal') + + def remove_all_ugly(self): + ''' + call remove_all() with in argument : "ugly" + ''' + self._remove_all('ugly') + + def _remove_all(self, method): + ''' + Remove all meta, normal way if method is "normal", + else, use the ugly way (with possible data loss) + ''' + raise NotImplementedError + + +class ZipStripper(GenericArchiveStripper): + ''' + Represent a zip file + ''' + def is_file_clean(self, fileinfo): + ''' + Check if a ZipInfo object is clean of metadatas added + by zip itself, independently of the corresponding file metadatas + ''' + if fileinfo.comment is not '': + return False + elif fileinfo.date_time is not 0: + return False + elif fileinfo.create_system is not 0: + return False + elif fileinfo.create_version is not 0: + return False + else: + return True + + def is_clean(self): + ''' + Check if the given file is clean from harmful metadata + ''' + zipin = zipfile.ZipFile(self.filename, 'r') + if zipin.comment != '': + logging.debug('%s has a comment' % self.filename) + return False + for item in zipin.infolist(): + #I have not found a way to remove the crap added by zipfile :/ + #if not self.is_file_clean(item): + # logging.debug('%s from %s has compromizing zipinfo' % + # (item.filename, self.filename)) + # return False + zipin.extract(item, self.tempdir) + name = os.path.join(self.tempdir, item.filename) + if os.path.isfile(name): + try: + cfile = mat.create_class_file(name, False, + self.add2archive) + if not cfile.is_clean(): + return False + except: + #best solution I have found + logging.info('%s\'s fileformat is not supported, or is a \ +harmless format' % item.filename) + _, ext = os.path.splitext(name) + bname = os.path.basename(item.filename) + if ext not in parser.NOMETA: + if bname != 'mimetype' and bname != '.rels': + return False + zipin.close() + return True + + def get_meta(self): + ''' + Return all the metadata of a ZipFile (don't return metadatas + of contained files : should it ?) + ''' + zipin = zipfile.ZipFile(self.filename, 'r') + metadata = {} + for field in zipin.infolist(): + zipmeta = {} + zipmeta['comment'] = field.comment + zipmeta['modified'] = field.date_time + zipmeta['system'] = field.create_system + zipmeta['zip_version'] = field.create_version + metadata[field.filename] = zipmeta + metadata["%s comment" % self.filename] = zipin.comment + zipin.close() + return metadata + + def _remove_all(self, method): + ''' + So far, the zipfile module does not allow to write a ZipInfo + object into a zipfile (and it's a shame !) : so data added + by zipfile itself could not be removed. It's a big concern. + Is shiping a patched version of zipfile.py a good idea ? + ''' + zipin = zipfile.ZipFile(self.filename, 'r') + zipout = zipfile.ZipFile(self.output, 'w', allowZip64=True) + for item in zipin.infolist(): + zipin.extract(item, self.tempdir) + name = os.path.join(self.tempdir, item.filename) + if os.path.isfile(name): + try: + cfile = mat.create_class_file(name, False, + self.add2archive) + if method is 'normal': + cfile.remove_all() + else: + cfile.remove_all_ugly() + logging.debug('Processing %s from %s' % (item.filename, + self.filename)) + zipout.write(name, item.filename) + except: + logging.info('%s\'s format is not supported or harmless' % + item.filename) + _, ext = os.path.splitext(name) + if self.add2archive or ext in parser.NOMETA: + zipout.write(name, item.filename) + zipout.comment = '' + zipin.close() + zipout.close() + logging.info('%s treated' % self.filename) + self.do_backup() + + +class TarStripper(GenericArchiveStripper): + ''' + Represent a tarfile archive + ''' + def _remove(self, current_file): + ''' + remove the meta added by tar itself to the file + ''' + current_file.mtime = 0 + current_file.uid = 0 + current_file.gid = 0 + current_file.uname = '' + current_file.gname = '' + return current_file + + def _remove_all(self, method): + tarin = tarfile.open(self.filename, 'r' + self.compression) + tarout = tarfile.open(self.output, 'w' + self.compression) + for item in tarin.getmembers(): + tarin.extract(item, self.tempdir) + name = os.path.join(self.tempdir, item.name) + if item.type is '0': # is item a regular file ? + #no backup file + try: + cfile = mat.create_class_file(name, False, + self.add2archive) + if method is 'normal': + cfile.remove_all() + else: + cfile.remove_all_ugly() + tarout.add(name, item.name, filter=self._remove) + except: + logging.info('%s\' format is not supported or harmless' % + item.name) + _, ext = os.path.splitext(name) + if self.add2archive or ext in parser.NOMETA: + tarout.add(name, item.name, filter=self._remove) + tarin.close() + tarout.close() + self.do_backup() + + def is_file_clean(self, current_file): + ''' + Check metadatas added by tar + ''' + if current_file.mtime is not 0: + return False + elif current_file.uid is not 0: + return False + elif current_file.gid is not 0: + return False + elif current_file.uname is not '': + return False + elif current_file.gname is not '': + return False + else: + return True + + def is_clean(self): + ''' + Check if the file is clean from harmful metadatas + ''' + tarin = tarfile.open(self.filename, 'r' + self.compression) + for item in tarin.getmembers(): + if not self.is_file_clean(item): + tarin.close() + return False + tarin.extract(item, self.tempdir) + name = os.path.join(self.tempdir, item.name) + if item.type is '0': # is item a regular file ? + try: + class_file = mat.create_class_file(name, + False, self.add2archive) # no backup file + if not class_file.is_clean(): + tarin.close() + return False + except: + logging.error('%s\'s foramt is not supported or harmless' % + item.filename) + _, ext = os.path.splitext(name) + if ext not in parser.NOMETA: + tarin.close() + return False + tarin.close() + return True + + def get_meta(self): + ''' + Return a dict with all the meta of the file + ''' + tarin = tarfile.open(self.filename, 'r' + self.compression) + metadata = {} + for current_file in tarin.getmembers(): + if current_file.type is '0': + if not self.is_file_clean(current_file): # if there is meta + current_meta = {} + current_meta['mtime'] = current_file.mtime + current_meta['uid'] = current_file.uid + current_meta['gid'] = current_file.gid + current_meta['uname'] = current_file.uname + current_meta['gname'] = current_file.gname + metadata[current_file.name] = current_meta + tarin.close() + return metadata + + +class GzipStripper(TarStripper): + ''' + Represent a tar.gz archive + ''' + def __init__(self, filename, parser, mime, backup, add2archive): + super(GzipStripper, self).__init__(filename, parser, mime, backup, + add2archive) + self.compression = ':gz' + + +class Bzip2Stripper(TarStripper): + ''' + Represents a tar.bz2 archive + ''' + def __init__(self, filename, parser, mime, backup, add2archive): + super(Bzip2Stripper, self).__init__(filename, parser, mime, backup, + add2archive) + self.compression = ':bz2' diff --git a/mat/audio.py b/mat/audio.py new file mode 100644 index 0000000..21a94be --- /dev/null +++ b/mat/audio.py @@ -0,0 +1,98 @@ +''' + Care about audio fileformat +''' +try: + from mutagen.flac import FLAC + from mutagen.oggvorbis import OggVorbis +except ImportError: + pass + + +import parser +import shutil + + +class MpegAudioStripper(parser.GenericParser): + ''' + Represent mpeg audio file (mp3, ...) + ''' + def _should_remove(self, field): + if field.name in ("id3v1", "id3v2"): + return True + else: + return False + + +class OggStripper(parser.GenericParser): + ''' + Represent an ogg vorbis file + ''' + def remove_all(self): + if self.backup is True: + shutil.copy2(self.filename, self.output) + self.filename = self.output + + mfile = OggVorbis(self.filename) + mfile.delete() + mfile.save() + + def is_clean(self): + ''' + Check if the "metadata" block is present in the file + ''' + mfile = OggVorbis(self.filename) + if mfile.tags == []: + return True + else: + return False + + def get_meta(self): + ''' + Return the content of the metadata block if present + ''' + metadata = {} + mfile = OggVorbis(self.filename) + for key, value in mfile.tags: + metadata[key] = value + return metadata + + +class FlacStripper(parser.GenericParser): + ''' + Represent a Flac audio file + ''' + def remove_all(self): + ''' + Remove the "metadata" block from the file + ''' + if self.backup is True: + shutil.copy2(self.filename, self.output) + self.filename = self.output + + mfile = FLAC(self.filename) + mfile.delete() + mfile.clear_pictures() + mfile.save() + + def is_clean(self): + ''' + Check if the "metadata" block is present in the file + ''' + mfile = FLAC(self.filename) + if mfile.tags is None and mfile.pictures == []: + return True + else: + return False + + def get_meta(self): + ''' + Return the content of the metadata block if present + ''' + metadata = {} + mfile = FLAC(self.filename) + if mfile.tags is not None: + if mfile.pictures != []: + metadata['picture :'] = 'yes' + for key, value in mfile.tags: + metadata[key] = value + return metadata diff --git a/mat/images.py b/mat/images.py new file mode 100644 index 0000000..d090015 --- /dev/null +++ b/mat/images.py @@ -0,0 +1,37 @@ +''' + Takes care about pictures formats +''' + +import parser + + +class JpegStripper(parser.GenericParser): + ''' + represents a jpeg file + ''' + def _should_remove(self, field): + ''' + return True if the field is compromizing + ''' + if field.name.startswith('comment'): + return True + elif field.name in ("photoshop", "exif", "adobe"): + return True + else: + return False + + +class PngStripper(parser.GenericParser): + ''' + represents a png file + ''' + def _should_remove(self, field): + ''' + return True if the field is compromizing + ''' + if field.name.startswith("text["): + return True + elif field.name is "time": + return True + else: + return False diff --git a/mat/mat.py b/mat/mat.py new file mode 100644 index 0000000..fd13287 --- /dev/null +++ b/mat/mat.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python + +''' + Metadata anonymisation toolkit library +''' + +import os +import subprocess +import logging +import mimetypes +import xml.sax + +import hachoir_core.cmd_line +import hachoir_parser + +import images +import audio +import office +import archive +import misc + +__version__ = '0.1' +__author__ = 'jvoisin' + +LOGGING_LEVEL = logging.DEBUG + +logging.basicConfig(level=LOGGING_LEVEL) + +STRIPPERS = { + 'application/x-tar': archive.TarStripper, + 'application/x-gzip': archive.GzipStripper, + 'application/x-bzip2': archive.Bzip2Stripper, + 'application/zip': archive.ZipStripper, + 'audio/mpeg': audio.MpegAudioStripper, + 'image/jpeg': images.JpegStripper, + 'image/png': images.PngStripper, + 'application/x-bittorrent': misc.TorrentStripper, + 'application/opendocument': office.OpenDocumentStripper, + 'application/officeopenxml': office.OpenXmlStripper, +} + +try: + import poppler + import cairo + STRIPPERS['application/x-pdf'] = office.PdfStripper + STRIPPERS['application/pdf'] = office.PdfStripper +except ImportError: + print('Unable to import python-poppler and/or python-cairo: no pdf \ + support') + +try: + import mutagen + STRIPPERS['audio/x-flac'] = audio.FlacStripper + STRIPPERS['audio/vorbis'] = audio.OggStripper +except ImportError: + print('unable to import python-mutagen : limited audio format support') + + +class XMLParser(xml.sax.handler.ContentHandler): + ''' + Parse the supported format xml, and return a corresponding + list of dict + ''' + def __init__(self): + self.dict = {} + self.list = [] + self.content, self.key = '', '' + self.between = False + + def startElement(self, name, attrs): + ''' + Called when entering into xml balise + ''' + self.between = True + self.key = name + self.content = '' + + def endElement(self, name): + ''' + Called when exiting a xml balise + ''' + if name == 'format': # exiting a fileformat section + self.list.append(self.dict.copy()) + self.dict.clear() + else: + content = self.content.replace('\s', ' ') + self.dict[self.key] = content + self.between = False + + def characters(self, characters): + ''' + Concatenate the content between opening and closing balises + ''' + if self.between is True: + self.content += characters + + +def secure_remove(filename): + ''' + securely remove the file + ''' + removed = False + try: + subprocess.call('shred --remove %s' % filename, shell=True) + removed = True + except: + logging.error('Unable to securely remove %s' % filename) + + if removed is False: + try: + os.remove(filename) + except: + logging.error('Unable to remove %s' % filename) + + +def is_secure(filename): + ''' + Prevent shell injection + ''' + if not(os.path.isfile(filename)): # check if the file exist + logging.error('%s is not a valid file' % filename) + return False + else: + return True + + +def create_class_file(name, backup, add2archive): + ''' + return a $FILETYPEStripper() class, + corresponding to the filetype of the given file + ''' + if not is_secure(name): + return + + filename = '' + try: + filename = hachoir_core.cmd_line.unicodeFilename(name) + except TypeError: # get rid of "decoding Unicode is not supported" + filename = name + + parser = hachoir_parser.createParser(filename) + if not parser: + logging.info('Unable to parse %s' % filename) + return + + mime = parser.mime_type + + if mime == 'application/zip': # some formats are zipped stuff + mime = mimetypes.guess_type(name)[0] + + if mime.startswith('application/vnd.oasis.opendocument'): + mime = 'application/opendocument' # opendocument fileformat + elif mime.startswith('application/vnd.openxmlformats-officedocument'): + mime = 'application/officeopenxml' # office openxml + + try: + stripper_class = STRIPPERS[mime] + except KeyError: + logging.info('Don\'t have stripper for %s format' % mime) + return + + return stripper_class(filename, parser, mime, backup, add2archive) diff --git a/mat/misc.py b/mat/misc.py new file mode 100644 index 0000000..f7b256f --- /dev/null +++ b/mat/misc.py @@ -0,0 +1,62 @@ +''' + Care about misc formats +''' + +import parser + +from bencode import bencode + + +class TorrentStripper(parser.GenericParser): + ''' + Represent a torrent file with the help + of the bencode lib from Petru Paler + ''' + def __init__(self, filename, parser, mime, backup, add2archive): + super(TorrentStripper, self).__init__(filename, parser, mime, + backup, add2archive) + self.fields = ['comment', 'creation date', 'created by'] + + def is_clean(self): + ''' + Check if the file is clean from harmful metadatas + ''' + with open(self.filename, 'r') as f: + decoded = bencode.bdecode(f.read()) + for key in self.fields: + try: + if decoded[key] != '': + return False + except: + pass + return True + + def get_meta(self): + ''' + Return a dict with all the meta of the file + ''' + metadata = {} + with open(self.filename, 'r') as f: + decoded = bencode.bdecode(f.read()) + for key in self.fields: + try: + if decoded[key] != '': + metadata[key] = decoded[key] + except: + pass + return metadata + + def remove_all(self): + ''' + Remove all the files that are compromizing + ''' + with open(self.filename, 'r') as f: + decoded = bencode.bdecode(f.read()) + for key in self.fields: + try: + decoded[key] = '' + except: + pass + with open(self.output, 'w') as f: # encode the decoded torrent + f.write(bencode.bencode(decoded)) # and write it in self.output + self.do_backup() diff --git a/mat/office.py b/mat/office.py new file mode 100644 index 0000000..cb9c609 --- /dev/null +++ b/mat/office.py @@ -0,0 +1,280 @@ +''' + Care about office's formats +''' + +import os +import logging +import zipfile +import fileinput + +try: + import cairo + import poppler +except ImportError: + pass + +import mat +import parser +import archive +import pdfrw + + +class OpenDocumentStripper(archive.GenericArchiveStripper): + ''' + An open document file is a zip, with xml file into. + The one that interest us is meta.xml + ''' + + def get_meta(self): + ''' + Return a dict with all the meta of the file by + trying to read the meta.xml file. + ''' + zipin = zipfile.ZipFile(self.filename, 'r') + metadata = {} + try: + content = zipin.read('meta.xml') + zipin.close() + metadata[self.filename] = 'harful meta' + except KeyError: # no meta.xml file found + logging.debug('%s has no opendocument metadata' % self.filename) + return metadata + + def _remove_all(self, method): + ''' + FIXME ? + There is a patch implementing the Zipfile.remove() + method here : http://bugs.python.org/issue6818 + ''' + zipin = zipfile.ZipFile(self.filename, 'r') + zipout = zipfile.ZipFile(self.output, 'w', allowZip64=True) + + for item in zipin.namelist(): + name = os.path.join(self.tempdir, item) + _, ext = os.path.splitext(name) + + if item.endswith('manifest.xml'): + # contain the list of all files present in the archive + zipin.extract(item, self.tempdir) + for line in fileinput.input(name, inplace=1): + #remove the line which contains "meta.xml" + line = line.strip() + if not 'meta.xml' in line: + print line + zipout.write(name, item) + + elif ext in parser.NOMETA or item == 'mimetype': + #keep NOMETA files, and the "manifest" file + if item != 'meta.xml': # contains the metadata + zipin.extract(item, self.tempdir) + zipout.write(name, item) + + else: + zipin.extract(item, self.tempdir) + if os.path.isfile(name): + try: + cfile = mat.create_class_file(name, False, + self.add2archive) + if method == 'normal': + cfile.remove_all() + else: + cfile.remove_all_ugly() + logging.debug('Processing %s from %s' % (item, + self.filename)) + zipout.write(name, item) + except: + logging.info('%s\' fileformat is not supported' % item) + if self.add2archive: + zipout.write(name, item) + zipout.comment = '' + logging.info('%s treated' % self.filename) + zipin.close() + zipout.close() + self.do_backup() + + def is_clean(self): + ''' + Check if the file is clean from harmful metadatas + ''' + zipin = zipfile.ZipFile(self.filename, 'r') + try: + zipin.getinfo('meta.xml') + except KeyError: # no meta.xml in the file + czf = archive.ZipStripper(self.filename, self.parser, + 'application/zip', self.backup, self.add2archive) + if czf.is_clean(): + zipin.close() + return True + zipin.close() + return False + + +class PdfStripper(parser.GenericParser): + ''' + Represent a pdf file + ''' + def __init__(self, filename, parser, mime, backup, add2archive): + super(PdfStripper, self).__init__(filename, parser, mime, backup, + add2archive) + uri = 'file://' + os.path.abspath(self.filename) + self.password = None + self.document = poppler.document_new_from_file(uri, self.password) + self.meta_list = ('title', 'author', 'subject', 'keywords', 'creator', + 'producer', 'creation-date', 'mod-date', 'metadata') + + def is_clean(self): + ''' + Check if the file is clean from harmful metadatas + ''' + for key in self.meta_list: + if key == 'creation-date' or key == 'mod-date': + if self.document.get_property(key) != -1: + return False + elif self.document.get_property(key) is not None and \ + self.document.get_property(key) != '': + return False + return True + + def remove_all_ugly(self): + page = self.document.get_page(0) + page_width, page_height = page.get_size() + surface = cairo.PDFSurface(self.output, page_width, page_height) + context = cairo.Context(surface) # context draws on the surface + logging.debug('Pdf rendering of %s' % self.filename) + for pagenum in xrange(self.document.get_n_pages()): + page = self.document.get_page(pagenum) + context.translate(0, 0) + page.render(context) # render the page on context + context.show_page() # draw context on surface + surface.finish() + + #For now, poppler cannot write meta, so we must use pdfrw + logging.debug('Removing %s\'s superficial metadata' % self.filename) + trailer = pdfrw.PdfReader(self.output) + trailer.Info.Producer = trailer.Info.Creator = None + writer = pdfrw.PdfWriter() + writer.trailer = trailer + writer.write(self.output) + self.do_backup() + + + def remove_all(self): + ''' + Opening the pdf with poppler, then doing a render + on a cairo pdfsurface for each pages. + Thanks to Lunar^for the idea. + http://cairographics.org/documentation/pycairo/2/ + python-poppler is not documented at all : have fun ;) + ''' + page = self.document.get_page(0) + page_width, page_height = page.get_size() + surface = cairo.PDFSurface(self.output, page_width, page_height) + context = cairo.Context(surface) # context draws on the surface + logging.debug('Pdf rendering of %s' % self.filename) + for pagenum in xrange(self.document.get_n_pages()): + page = self.document.get_page(pagenum) + context.translate(0, 0) + page.render(context) # render the page on context + context.show_page() # draw context on surface + surface.finish() + + #For now, poppler cannot write meta, so we must use pdfrw + logging.debug('Removing %s\'s superficial metadata' % self.filename) + trailer = pdfrw.PdfReader(self.output) + trailer.Info.Producer = trailer.Info.Creator = None + writer = pdfrw.PdfWriter() + writer.trailer = trailer + writer.write(self.output) + self.do_backup() + + def get_meta(self): + ''' + Return a dict with all the meta of the file + ''' + metadata = {} + for key in self.meta_list: + if key == 'creation-date' or key == 'mod-date': + #creation and modification are set to -1 + if self.document.get_property(key) != -1: + metadata[key] = self.document.get_property(key) + elif self.document.get_property(key) is not None and \ + self.document.get_property(key) != '': + metadata[key] = self.document.get_property(key) + return metadata + + +class OpenXmlStripper(archive.GenericArchiveStripper): + ''' + Represent an office openxml document, which is like + an opendocument format, with some tricky stuff added. + It contains mostly xml, but can have media blobs, crap, ... + (I don't like this format.) + ''' + def _remove_all(self, method): + ''' + FIXME ? + There is a patch implementing the Zipfile.remove() + method here : http://bugs.python.org/issue6818 + ''' + zipin = zipfile.ZipFile(self.filename, 'r') + zipout = zipfile.ZipFile(self.output, 'w', + allowZip64=True) + for item in zipin.namelist(): + name = os.path.join(self.tempdir, item) + _, ext = os.path.splitext(name) + if item.startswith('docProps/'): # metadatas + pass + elif ext in parser.NOMETA or item == '.rels': + #keep parser.NOMETA files, and the file named ".rels" + zipin.extract(item, self.tempdir) + zipout.write(name, item) + else: + zipin.extract(item, self.tempdir) + if os.path.isfile(name): # don't care about folders + try: + cfile = mat.create_class_file(name, False, + self.add2archive) + if method == 'normal': + cfile.remove_all() + else: + cfile.remove_all_ugly() + logging.debug('Processing %s from %s' % (item, + self.filename)) + zipout.write(name, item) + except: + logging.info('%s\' fileformat is not supported' % item) + if self.add2archive: + zipout.write(name, item) + zipout.comment = '' + logging.info('%s treated' % self.filename) + zipin.close() + zipout.close() + self.do_backup() + + def is_clean(self): + ''' + Check if the file is clean from harmful metadatas + ''' + zipin = zipfile.ZipFile(self.filename, 'r') + for item in zipin.namelist(): + if item.startswith('docProps/'): + return False + zipin.close() + czf = archive.ZipStripper(self.filename, self.parser, + 'application/zip', self.backup, self.add2archive) + if not czf.is_clean(): + return False + else: + return True + + def get_meta(self): + ''' + Return a dict with all the meta of the file + ''' + zipin = zipfile.ZipFile(self.filename, 'r') + metadata = {} + for item in zipin.namelist(): + if item.startswith('docProps/'): + metadata[item] = 'harmful content' + zipin.close() + return metadata diff --git a/mat/parser.py b/mat/parser.py new file mode 100644 index 0000000..58dd7fa --- /dev/null +++ b/mat/parser.py @@ -0,0 +1,104 @@ +''' + Parent class of all parser +''' + +import hachoir_core +import hachoir_editor + +import os + +import mat + +NOMETA = ('.bmp', '.rdf', '.txt', '.xml', '.rels') +#bmp : image +#rdf : text +#txt : plain text +#xml : formated text +#rels : openxml foramted text + + +class GenericParser(object): + ''' + Parent class of all parsers + ''' + def __init__(self, filename, parser, mime, backup, add2archive): + self.filename = '' + self.parser = parser + self.mime = mime + self.backup = backup + self.editor = hachoir_editor.createEditor(parser) + self.realname = filename + try: + self.filename = hachoir_core.cmd_line.unicodeFilename(filename) + except TypeError: # get rid of "decoding Unicode is not supported" + self.filename = filename + basename, ext = os.path.splitext(filename) + self.output = basename + '.cleaned' + ext + self.basename = os.path.basename(filename) # only filename + + def is_clean(self): + ''' + Check if the file is clean from harmful metadatas + ''' + for field in self.editor: + if self._should_remove(field): + return False + return True + + def remove_all(self): + ''' + Remove all the files that are compromizing + ''' + for field in self.editor: + if self._should_remove(field): + self._remove(field.name) + hachoir_core.field.writeIntoFile(self.editor, self.output) + self.do_backup() + + def remove_all_ugly(self): + ''' + If the remove_all() is not efficient enough, + this method is implemented : + It is efficient, but destructive. + In a perfect world, with nice fileformat, + this method would not exist. + ''' + self.remove_all() + + def _remove(self, field): + ''' + Delete the given field + ''' + del self.editor[field] + + def get_meta(self): + ''' + Return a dict with all the meta of the file + ''' + metadata = {} + for field in self.editor: + if self._should_remove(field): + try: + metadata[field.name] = field.value + except: + metadata[field.name] = 'harmful content' + return metadata + + def _should_remove(self, key): + ''' + return True if the field is compromizing + abstract method + ''' + raise NotImplementedError + + def do_backup(self): + ''' + Do a backup of the file if asked, + and change his creation/access date + ''' + if self.backup is True: + os.utime(self.output, (0, 0)) + else: + mat.secure_remove(self.filename) + os.rename(self.output, self.filename) + os.utime(self.filename, (0, 0)) diff --git a/setup.py b/setup.py index fc1796f..35d282e 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + from distutils.core import setup setup(name='MAT', diff --git a/test/clitest.py b/test/clitest.py index 453333a..781fb80 100644 --- a/test/clitest.py +++ b/test/clitest.py @@ -8,7 +8,7 @@ import subprocess import sys sys.path.append('..') -from lib import mat +from mat import mat import test class TestRemovecli(test.MATTest): diff --git a/test/libtest.py b/test/libtest.py index 2390841..d8aaac1 100644 --- a/test/libtest.py +++ b/test/libtest.py @@ -8,7 +8,7 @@ import unittest import test import sys sys.path.append('..') -from lib import mat +from mat import mat class TestRemovelib(test.MATTest): ''' -- cgit v1.3