From cbf8a2a65928694202e19b6bcf56ec84bcbf613c Mon Sep 17 00:00:00 2001
From: jvoisin
Date: Sat, 8 Dec 2012 02:02:25 +0100
Subject: Reorganize source tree and files installation location, cleanup
setup.py (Closes: #689409)
---
MAT/FORMATS | 94 ++
MAT/__init__.py | 1 +
MAT/archive.py | 272 ++++
MAT/audio.py | 100 ++
MAT/bencode/__init__.py | 1 +
MAT/bencode/bencode.py | 152 +++
MAT/exiftool.py | 95 ++
MAT/hachoir_editor/__init__.py | 8 +
MAT/hachoir_editor/field.py | 69 +
MAT/hachoir_editor/fieldset.py | 352 +++++
MAT/hachoir_editor/typed_field.py | 253 ++++
MAT/images.py | 41 +
MAT/mat.py | 152 +++
MAT/misc.py | 63 +
MAT/office.py | 265 ++++
MAT/parser.py | 120 ++
MAT/strippers.py | 67 +
MAT/tarfile/__init__.py | 1 +
MAT/tarfile/tarfile.py | 2593 +++++++++++++++++++++++++++++++++++++
19 files changed, 4699 insertions(+)
create mode 100644 MAT/FORMATS
create mode 100644 MAT/__init__.py
create mode 100644 MAT/archive.py
create mode 100644 MAT/audio.py
create mode 100644 MAT/bencode/__init__.py
create mode 100644 MAT/bencode/bencode.py
create mode 100644 MAT/exiftool.py
create mode 100644 MAT/hachoir_editor/__init__.py
create mode 100644 MAT/hachoir_editor/field.py
create mode 100644 MAT/hachoir_editor/fieldset.py
create mode 100644 MAT/hachoir_editor/typed_field.py
create mode 100644 MAT/images.py
create mode 100644 MAT/mat.py
create mode 100644 MAT/misc.py
create mode 100644 MAT/office.py
create mode 100644 MAT/parser.py
create mode 100644 MAT/strippers.py
create mode 100644 MAT/tarfile/__init__.py
create mode 100644 MAT/tarfile/tarfile.py
(limited to 'MAT')
diff --git a/MAT/FORMATS b/MAT/FORMATS
new file mode 100644
index 0000000..c497524
--- /dev/null
+++ b/MAT/FORMATS
@@ -0,0 +1,94 @@
+
+
+ Portable Network Graphics
+ .png
+ full
+ textual metadata + date
+ removal of harmful fields is done with hachoir
+
+
+
+ Jpeg
+ .jpeg, .jpg
+ full
+ comment + exif/photoshop/adobe
+ removal of harmful fields is done with hachoir
+
+
+
+ Open Document
+ .odt, .odx, .ods, ...
+ full
+ a meta.xml file
+ removal of the meta.xml file
+
+
+
+ Office Openxml
+ .docx, .pptx, .xlsx, ...
+ full
+ a docProps folder containings xml metadata files
+ removal of the docProps folder
+
+
+
+ Portable Document Fileformat
+ .pdf
+ full
+ a lot
+ rendering of the pdf file on a cairo surface with the help of
+ poppler in order to remove all the internal metadata,
+ then removal of the remaining metadata fields of the pdf itself with
+ pdfrw (the next version of python-cairo will support metadata,
+ so we should get rid of pdfrw)
+
+
+
+ Tape ARchive
+ .tar, .tar.bz2, .tar.gz
+ full
+ metadata from the file itself, metadata from the file contained
+ into the archive, and metadata added by tar to the file at then
+ creation of the archive
+ extraction of each file, treatement of the file, add treated file
+ to a new archive, right before the add, remove the metadata added by tar
+ itself. When the new archive is complete, remove all his metadata.
+
+
+
+ Zip
+ .zip
+ .partial
+ metadata from the file itself, metadata from the file contained
+ into the archive, and metadata added by zip to the file when added to
+ the archive.
+
+ extraction of each file, treatement of the file, add treated file
+ to a new archive. When the new archive is complete, remove all his metadata
+ metadata added by zip itself to internal files
+
+
+
+ MPEG Audio
+ .mp3, .mp2, .mp1
+ full
+ id3
+ removal of harmful fields is done with hachoir
+
+
+
+ Ogg Vorbis
+ .ogg
+ full
+ Vorbis
+ removal of harmful fields is done with mutagen
+
+
+
+ Free Lossless Audio Codec
+ .flac
+ full
+ Flac, Vorbis
+ removal of harmful fields is done with mutagen
+
+
diff --git a/MAT/__init__.py b/MAT/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/MAT/__init__.py
@@ -0,0 +1 @@
+
diff --git a/MAT/archive.py b/MAT/archive.py
new file mode 100644
index 0000000..a749b29
--- /dev/null
+++ b/MAT/archive.py
@@ -0,0 +1,272 @@
+'''
+ Take care of archives formats
+'''
+
+import zipfile
+import shutil
+import os
+import logging
+import tempfile
+
+import parser
+import mat
+from tarfile import tarfile
+
+
+class GenericArchiveStripper(parser.GenericParser):
+ '''
+ Represent a generic archive
+ '''
+ def __init__(self, filename, parser, mime, backup, add2archive):
+ super(GenericArchiveStripper, self).__init__(filename, parser, mime,
+ backup, add2archive)
+ self.compression = ''
+ self.add2archive = add2archive
+ self.tempdir = tempfile.mkdtemp()
+
+ def __del__(self):
+ '''
+ Remove the files inside the temp dir,
+ then remove the temp dir
+ '''
+ for root, dirs, files in os.walk(self.tempdir):
+ for item in files:
+ path_file = os.path.join(root, item)
+ mat.secure_remove(path_file)
+ shutil.rmtree(self.tempdir)
+
+ def remove_all(self):
+ return self._remove_all()
+
+ def _remove_all(self):
+ raise NotImplementedError
+
+
+class ZipStripper(GenericArchiveStripper):
+ '''
+ Represent a zip file
+ '''
+ def is_file_clean(self, fileinfo):
+ '''
+ Check if a ZipInfo object is clean of metadatas added
+ by zip itself, independently of the corresponding file metadatas
+ '''
+ if fileinfo.comment is not '':
+ return False
+ elif fileinfo.date_time is not 0:
+ return False
+ elif fileinfo.create_system is not 0:
+ return False
+ elif fileinfo.create_version is not 0:
+ return False
+ else:
+ return True
+
+ def is_clean(self):
+ '''
+ Check if the given file is clean from harmful metadata
+ '''
+ zipin = zipfile.ZipFile(self.filename, 'r')
+ if zipin.comment != '':
+ logging.debug('%s has a comment' % self.filename)
+ return False
+ for item in zipin.infolist():
+ #I have not found a way to remove the crap added by zipfile :/
+ #if not self.is_file_clean(item):
+ # logging.debug('%s from %s has compromizing zipinfo' %
+ # (item.filename, self.filename))
+ # return False
+ zipin.extract(item, self.tempdir)
+ name = os.path.join(self.tempdir, item.filename)
+ if os.path.isfile(name):
+ try:
+ cfile = mat.create_class_file(name, False,
+ self.add2archive)
+ if not cfile.is_clean():
+ return False
+ except:
+ #best solution I have found
+ logging.info('%s\'s fileformat is not supported, or is a \
+harmless format' % item.filename)
+ _, ext = os.path.splitext(name)
+ bname = os.path.basename(item.filename)
+ if ext not in parser.NOMETA:
+ if bname != 'mimetype' and bname != '.rels':
+ return False
+ zipin.close()
+ return True
+
+ def get_meta(self):
+ '''
+ Return all the metadata of a ZipFile (don't return metadatas
+ of contained files : should it ?)
+ '''
+ zipin = zipfile.ZipFile(self.filename, 'r')
+ metadata = {}
+ for field in zipin.infolist():
+ zipmeta = {}
+ zipmeta['comment'] = field.comment
+ zipmeta['modified'] = field.date_time
+ zipmeta['system'] = field.create_system
+ zipmeta['zip_version'] = field.create_version
+ metadata[field.filename] = zipmeta
+ metadata["%s comment" % self.filename] = zipin.comment
+ zipin.close()
+ return metadata
+
+ def _remove_all(self):
+ '''
+ So far, the zipfile module does not allow to write a ZipInfo
+ object into a zipfile (and it's a shame !) : so data added
+ by zipfile itself could not be removed. It's a big concern.
+ Is shiping a patched version of zipfile.py a good idea ?
+ '''
+ zipin = zipfile.ZipFile(self.filename, 'r')
+ zipout = zipfile.ZipFile(self.output, 'w', allowZip64=True)
+ for item in zipin.infolist():
+ zipin.extract(item, self.tempdir)
+ name = os.path.join(self.tempdir, item.filename)
+ if os.path.isfile(name):
+ try:
+ cfile = mat.create_class_file(name, False,
+ self.add2archive)
+ cfile.remove_all()
+ logging.debug('Processing %s from %s' % (item.filename,
+ self.filename))
+ zipout.write(name, item.filename)
+ except:
+ logging.info('%s\'s format is not supported or harmless' %
+ item.filename)
+ _, ext = os.path.splitext(name)
+ if self.add2archive or ext in parser.NOMETA:
+ zipout.write(name, item.filename)
+ zipout.comment = ''
+ zipin.close()
+ zipout.close()
+ logging.info('%s treated' % self.filename)
+ self.do_backup()
+ return True
+
+
+class TarStripper(GenericArchiveStripper):
+ '''
+ Represent a tarfile archive
+ '''
+ def _remove(self, current_file):
+ '''
+ remove the meta added by tar itself to the file
+ '''
+ current_file.mtime = 0
+ current_file.uid = 0
+ current_file.gid = 0
+ current_file.uname = ''
+ current_file.gname = ''
+ return current_file
+
+ def _remove_all(self):
+ tarin = tarfile.open(self.filename, 'r' + self.compression)
+ tarout = tarfile.open(self.output, 'w' + self.compression)
+ for item in tarin.getmembers():
+ tarin.extract(item, self.tempdir)
+ name = os.path.join(self.tempdir, item.name)
+ if item.type is '0': # is item a regular file ?
+ #no backup file
+ try:
+ cfile = mat.create_class_file(name, False,
+ self.add2archive)
+ cfile.remove_all()
+ tarout.add(name, item.name, filter=self._remove)
+ except:
+ logging.info('%s\' format is not supported or harmless' %
+ item.name)
+ _, ext = os.path.splitext(name)
+ if self.add2archive or ext in parser.NOMETA:
+ tarout.add(name, item.name, filter=self._remove)
+ tarin.close()
+ tarout.close()
+ self.do_backup()
+ return True
+
+ def is_file_clean(self, current_file):
+ '''
+ Check metadatas added by tar
+ '''
+ if current_file.mtime is not 0:
+ return False
+ elif current_file.uid is not 0:
+ return False
+ elif current_file.gid is not 0:
+ return False
+ elif current_file.uname is not '':
+ return False
+ elif current_file.gname is not '':
+ return False
+ else:
+ return True
+
+ def is_clean(self):
+ '''
+ Check if the file is clean from harmful metadatas
+ '''
+ tarin = tarfile.open(self.filename, 'r' + self.compression)
+ for item in tarin.getmembers():
+ if not self.is_file_clean(item):
+ tarin.close()
+ return False
+ tarin.extract(item, self.tempdir)
+ name = os.path.join(self.tempdir, item.name)
+ if item.type is '0': # is item a regular file ?
+ try:
+ class_file = mat.create_class_file(name,
+ False, self.add2archive) # no backup file
+ if not class_file.is_clean():
+ tarin.close()
+ return False
+ except:
+ logging.error('%s\'s foramt is not supported or harmless' %
+ item.filename)
+ _, ext = os.path.splitext(name)
+ if ext not in parser.NOMETA:
+ tarin.close()
+ return False
+ tarin.close()
+ return True
+
+ def get_meta(self):
+ '''
+ Return a dict with all the meta of the file
+ '''
+ tarin = tarfile.open(self.filename, 'r' + self.compression)
+ metadata = {}
+ for current_file in tarin.getmembers():
+ if current_file.type is '0':
+ if not self.is_file_clean(current_file): # if there is meta
+ current_meta = {}
+ current_meta['mtime'] = current_file.mtime
+ current_meta['uid'] = current_file.uid
+ current_meta['gid'] = current_file.gid
+ current_meta['uname'] = current_file.uname
+ current_meta['gname'] = current_file.gname
+ metadata[current_file.name] = current_meta
+ tarin.close()
+ return metadata
+
+
+class GzipStripper(TarStripper):
+ '''
+ Represent a tar.gz archive
+ '''
+ def __init__(self, filename, parser, mime, backup, add2archive):
+ super(GzipStripper, self).__init__(filename, parser, mime, backup,
+ add2archive)
+ self.compression = ':gz'
+
+
+class Bzip2Stripper(TarStripper):
+ '''
+ Represents a tar.bz2 archive
+ '''
+ def __init__(self, filename, parser, mime, backup, add2archive):
+ super(Bzip2Stripper, self).__init__(filename, parser, mime, backup,
+ add2archive)
+ self.compression = ':bz2'
diff --git a/MAT/audio.py b/MAT/audio.py
new file mode 100644
index 0000000..ed849ee
--- /dev/null
+++ b/MAT/audio.py
@@ -0,0 +1,100 @@
+'''
+ Care about audio fileformat
+'''
+try:
+ from mutagen.flac import FLAC
+ from mutagen.oggvorbis import OggVorbis
+except ImportError:
+ pass
+
+
+import parser
+import shutil
+
+
+class MpegAudioStripper(parser.GenericParser):
+ '''
+ Represent mpeg audio file (mp3, ...)
+ '''
+ def _should_remove(self, field):
+ if field.name in ("id3v1", "id3v2"):
+ return True
+ else:
+ return False
+
+
+class OggStripper(parser.GenericParser):
+ '''
+ Represent an ogg vorbis file
+ '''
+ def remove_all(self):
+ if self.backup is True:
+ shutil.copy2(self.filename, self.output)
+ self.filename = self.output
+
+ mfile = OggVorbis(self.filename)
+ mfile.delete()
+ mfile.save()
+ return True
+
+ def is_clean(self):
+ '''
+ Check if the "metadata" block is present in the file
+ '''
+ mfile = OggVorbis(self.filename)
+ if mfile.tags == []:
+ return True
+ else:
+ return False
+
+ def get_meta(self):
+ '''
+ Return the content of the metadata block if present
+ '''
+ metadata = {}
+ mfile = OggVorbis(self.filename)
+ for key, value in mfile.tags:
+ metadata[key] = value
+ return metadata
+
+
+class FlacStripper(parser.GenericParser):
+ '''
+ Represent a Flac audio file
+ '''
+ def remove_all(self):
+ '''
+ Remove the "metadata" block from the file
+ '''
+ if self.backup is True:
+ shutil.copy2(self.filename, self.output)
+ self.filename = self.output
+
+ mfile = FLAC(self.filename)
+ mfile.delete()
+ mfile.clear_pictures()
+ mfile.save()
+ return True
+
+ def is_clean(self):
+ '''
+ Check if the "metadata" block is present in the file
+ '''
+ mfile = FLAC(self.filename)
+ if mfile.tags is None and mfile.pictures == []:
+ return True
+ else:
+ return False
+
+ def get_meta(self):
+ '''
+ Return the content of the metadata block if present
+ '''
+ metadata = {}
+ mfile = FLAC(self.filename)
+ if mfile.tags is not None:
+ if mfile.pictures != []:
+ metadata['picture :'] = 'yes'
+ for key, value in mfile.tags:
+ metadata[key] = value
+ return metadata
diff --git a/MAT/bencode/__init__.py b/MAT/bencode/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/MAT/bencode/__init__.py
@@ -0,0 +1 @@
+
diff --git a/MAT/bencode/bencode.py b/MAT/bencode/bencode.py
new file mode 100644
index 0000000..739ffe5
--- /dev/null
+++ b/MAT/bencode/bencode.py
@@ -0,0 +1,152 @@
+# Copyright 2007 by Petru Paler
+# Copyright 2011 by Julien (jvoisin) Voisin
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+'''
+ A quick (and also nice) lib to bencode/bdecode torrent files
+'''
+
+
+import types
+
+
+class BTFailure(Exception):
+ '''Custom Exception'''
+ pass
+
+
+class Bencached(object):
+ '''Custom type : cached string'''
+ __slots__ = ['bencoded']
+
+ def __init__(self, string):
+ self.bencoded = string
+
+
+def decode_int(x, f):
+ '''decode an int'''
+ f += 1
+ newf = x.index('e', f)
+ n = int(x[f:newf])
+ if x[f] == '-':
+ if x[f + 1] == '0':
+ raise ValueError
+ elif x[f] == '0' and newf != f + 1:
+ raise ValueError
+ return (n, newf + 1)
+
+
+def decode_string(x, f):
+ '''decode a string'''
+ colon = x.index(':', f)
+ n = int(x[f:colon])
+ if x[f] == '0' and colon != f + 1:
+ raise ValueError
+ colon += 1
+ return (x[colon:colon + n], colon + n)
+
+
+def decode_list(x, f):
+ '''decode a list'''
+ result = []
+ f += 1
+ while x[f] != 'e':
+ v, f = DECODE_FUNC[x[f]](x, f)
+ result.append(v)
+ return (result, f + 1)
+
+
+def decode_dict(x, f):
+ '''decode a dict'''
+ result = {}
+ f += 1
+ while x[f] != 'e':
+ k, f = decode_string(x, f)
+ result[k], f = DECODE_FUNC[x[f]](x, f)
+ return (result, f + 1)
+
+
+def encode_bool(x, r):
+ '''bencode a boolean'''
+ if x:
+ encode_int(1, r)
+ else:
+ encode_int(0, r)
+
+
+def encode_int(x, r):
+ '''bencode an integer/float'''
+ r.extend(('i', str(x), 'e'))
+
+
+def encode_list(x, r):
+ '''bencode a list/tuple'''
+ r.append('l')
+ [ENCODE_FUNC[type(item)](item, r) for item in x]
+ r.append('e')
+
+
+def encode_dict(x, result):
+ '''bencode a dict'''
+ result.append('d')
+ ilist = x.items()
+ ilist.sort()
+ for k, v in ilist:
+ result.extend((str(len(k)), ':', k))
+ ENCODE_FUNC[type(v)](v, result)
+ result.append('e')
+
+
+DECODE_FUNC = {}
+DECODE_FUNC.update(dict([(str(x), decode_string) for x in xrange(9)]))
+DECODE_FUNC['l'] = decode_list
+DECODE_FUNC['d'] = decode_dict
+DECODE_FUNC['i'] = decode_int
+
+
+ENCODE_FUNC = {}
+ENCODE_FUNC[Bencached] = lambda x, r: r.append(x.bencoded)
+ENCODE_FUNC[types.IntType] = encode_int
+ENCODE_FUNC[types.LongType] = encode_int
+ENCODE_FUNC[types.StringType] = lambda x, r: r.extend((str(len(x)), ':', x))
+ENCODE_FUNC[types.ListType] = encode_list
+ENCODE_FUNC[types.TupleType] = encode_list
+ENCODE_FUNC[types.DictType] = encode_dict
+ENCODE_FUNC[types.BooleanType] = encode_bool
+
+
+def bencode(string):
+ '''bencode $string'''
+ table = []
+ ENCODE_FUNC[type(string)](string, table)
+ return ''.join(table)
+
+
+def bdecode(string):
+ '''decode $string'''
+ try:
+ result, lenght = DECODE_FUNC[string[0]](string, 0)
+ except (IndexError, KeyError, ValueError):
+ raise BTFailure('Not a valid bencoded string')
+ if lenght != len(string):
+ raise BTFailure('Invalid bencoded value (data after valid prefix)')
+ return result
diff --git a/MAT/exiftool.py b/MAT/exiftool.py
new file mode 100644
index 0000000..758a094
--- /dev/null
+++ b/MAT/exiftool.py
@@ -0,0 +1,95 @@
+'''
+ Care about images with help of the amazing (perl) library Exiftool.
+'''
+
+import subprocess
+import parser
+
+
+class ExiftoolStripper(parser.GenericParser):
+ '''
+ A generic stripper class using exiftool as backend
+ '''
+
+ def __init__(self, filename, parser, mime, backup, add2archive):
+ super(ExiftoolStripper, self).__init__(filename, parser, mime,
+ backup, add2archive)
+ self.allowed = ['ExifTool Version Number', 'File Name', 'Directory',
+ 'File Size', 'File Modification Date/Time', 'File Permissions',
+ 'File Type', 'MIME Type', 'Image Width', 'Image Height',
+ 'Image Size']
+ self._set_allowed()
+
+ def _set_allowed(self):
+ '''
+ Set the allowed/harmless list of metadata
+ '''
+ raise NotImplementedError
+
+ def remove_all(self):
+ '''
+ Remove all metadata with help of exiftool
+ '''
+ try:
+ if self.backup:
+ # Note: '-All=' must be followed by a known exiftool option.
+ process = subprocess.Popen(['exiftool', '-m', '-All=',
+ '-out', self.output, self.filename],
+ stdout=open('/dev/null'))
+ process.wait()
+ else:
+ # Note: '-All=' must be followed by a known exiftool option.
+ process = subprocess.Popen(
+ [ 'exiftool', '-m', '-All=', '-overwrite_original', self.filename ],
+ stdout=open('/dev/null'))
+ process.wait()
+ return True
+ except:
+ return False
+
+ def is_clean(self):
+ '''
+ Check if the file is clean with help of exiftool
+ '''
+ out = subprocess.Popen(['exiftool', self.filename],
+ stdout=subprocess.PIPE).communicate()[0]
+ out = out.split('\n')
+ for i in out[:-1]:
+ if i.split(':')[0].strip() not in self.allowed:
+ return False
+ return True
+
+ def get_meta(self):
+ '''
+ Return every harmful meta with help of exiftool
+ '''
+ out = subprocess.Popen(['exiftool', self.filename],
+ stdout=subprocess.PIPE).communicate()[0]
+ out = out.split('\n')
+ meta = {}
+ for i in out[:-1]:
+ key = i.split(':')[0].strip()
+ if key not in self.allowed:
+ meta[key] = i.split(':')[1].strip()
+ return meta
+
+
+class JpegStripper(ExiftoolStripper):
+ '''
+ Care about jpeg files with help
+ of exiftool
+ '''
+ def _set_allowed(self):
+ self.allowed.extend(['JFIF Version', 'Resolution Unit',
+ 'X Resolution', 'Y Resolution', 'Encoding Process', 'Bits Per Sample',
+ 'Color Components', 'Y Cb Cr Sub Sampling'])
+
+class PngStripper(ExiftoolStripper):
+ '''
+ Care about png files with help
+ of exiftool
+ '''
+ def _set_allowed(self):
+ self.allowed.extend(['Bit Depth', 'Color Type', 'Compression',
+ 'Filter', 'Interlace', 'Pixels Per Unit X', 'Pixels Per Unit Y',
+ 'Pixel Units'])
diff --git a/MAT/hachoir_editor/__init__.py b/MAT/hachoir_editor/__init__.py
new file mode 100644
index 0000000..1835676
--- /dev/null
+++ b/MAT/hachoir_editor/__init__.py
@@ -0,0 +1,8 @@
+from field import (
+ EditorError, FakeField)
+from typed_field import (
+ EditableField, EditableBits, EditableBytes,
+ EditableInteger, EditableString,
+ createEditableField)
+from fieldset import EditableFieldSet, NewFieldSet, createEditor
+
diff --git a/MAT/hachoir_editor/field.py b/MAT/hachoir_editor/field.py
new file mode 100644
index 0000000..6b1efe3
--- /dev/null
+++ b/MAT/hachoir_editor/field.py
@@ -0,0 +1,69 @@
+from hachoir_core.error import HachoirError
+from hachoir_core.field import joinPath, MissingField
+
+class EditorError(HachoirError):
+ pass
+
+class FakeField(object):
+ """
+ This class have API looks similar to Field API, but objects don't contain
+ any value: all values are _computed_ by parent methods.
+
+ Example: FakeField(editor, "abc").size calls editor._getFieldSize("abc").
+ """
+ is_field_set = False
+
+ def __init__(self, parent, name):
+ self._parent = parent
+ self._name = name
+
+ def _getPath(self):
+ return joinPath(self._parent.path, self._name)
+ path = property(_getPath)
+
+ def _getName(self):
+ return self._name
+ name = property(_getName)
+
+ def _getAddress(self):
+ return self._parent._getFieldAddress(self._name)
+ address = property(_getAddress)
+
+ def _getSize(self):
+ return self._parent.input[self._name].size
+ size = property(_getSize)
+
+ def _getValue(self):
+ return self._parent.input[self._name].value
+ value = property(_getValue)
+
+ def createDisplay(self):
+ # TODO: Returns new value if field is altered
+ return self._parent.input[self._name].display
+ display = property(createDisplay)
+
+ def _getParent(self):
+ return self._parent
+ parent = property(_getParent)
+
+ def hasValue(self):
+ return self._parent.input[self._name].hasValue()
+
+ def __getitem__(self, key):
+ # TODO: Implement this function!
+ raise MissingField(self, key)
+
+ def _isAltered(self):
+ return False
+ is_altered = property(_isAltered)
+
+ def writeInto(self, output):
+ size = self.size
+ addr = self._parent._getFieldInputAddress(self._name)
+ input = self._parent.input
+ stream = input.stream
+ if size % 8:
+ output.copyBitsFrom(stream, addr, size, input.endian)
+ else:
+ output.copyBytesFrom(stream, addr, size//8)
+
diff --git a/MAT/hachoir_editor/fieldset.py b/MAT/hachoir_editor/fieldset.py
new file mode 100644
index 0000000..b7c9b07
--- /dev/null
+++ b/MAT/hachoir_editor/fieldset.py
@@ -0,0 +1,352 @@
+from hachoir_core.dict import UniqKeyError
+from hachoir_core.field import MissingField, Float32, Float64, FakeArray
+from hachoir_core.compatibility import any
+from hachoir_core.i18n import _
+from typed_field import createEditableField
+from field import EditorError
+from collections import deque # Python 2.4
+import weakref # Python 2.1
+import struct
+
+class EditableFieldSet(object):
+ MAX_SIZE = (1 << 40) # Arbitrary limit to catch errors
+ is_field_set = True
+
+ def __init__(self, parent, fieldset):
+ self._parent = parent
+ self.input = fieldset # original FieldSet
+ self._fields = {} # cache of editable fields
+ self._deleted = set() # Names of deleted fields
+ self._inserted = {} # Inserted field (name => list of field,
+ # where name is the name after)
+
+ def array(self, key):
+ # FIXME: Use cache?
+ return FakeArray(self, key)
+
+ def _getParent(self):
+ return self._parent
+ parent = property(_getParent)
+
+ def _isAltered(self):
+ if self._inserted:
+ return True
+ if self._deleted:
+ return True
+ return any(field.is_altered for field in self._fields.itervalues())
+ is_altered = property(_isAltered)
+
+ def reset(self):
+ """
+ Reset the field set and the input field set.
+ """
+ for key, field in self._fields.iteritems():
+ if not field.is_altered:
+ del self._fields[key]
+ self.input.reset()
+
+ def __len__(self):
+ return len(self.input) \
+ - len(self._deleted) \
+ + sum( len(new) for new in self._inserted.itervalues() )
+
+ def __iter__(self):
+ for field in self.input:
+ name = field.name
+ if name in self._inserted:
+ for newfield in self._inserted[name]:
+ yield weakref.proxy(newfield)
+ if name not in self._deleted:
+ yield self[name]
+ if None in self._inserted:
+ for newfield in self._inserted[None]:
+ yield weakref.proxy(newfield)
+
+ def insertBefore(self, name, *new_fields):
+ self._insert(name, new_fields, False)
+
+ def insertAfter(self, name, *new_fields):
+ self._insert(name, new_fields, True)
+
+ def insert(self, *new_fields):
+ self._insert(None, new_fields, True)
+
+ def _insert(self, key, new_fields, next):
+ """
+ key is the name of the field before which new_fields
+ will be inserted. If next is True, the fields will be inserted
+ _after_ this field.
+ """
+ # Set unique field name
+ for field in new_fields:
+ if field._name.endswith("[]"):
+ self.input.setUniqueFieldName(field)
+
+ # Check that there is no duplicate in inserted fields
+ new_names = list(field.name for field in new_fields)
+ names_set = set(new_names)
+ if len(names_set) != len(new_fields):
+ duplicates = (name for name in names_set if 1 < new_names.count(name))
+ raise UniqKeyError(_("Duplicates in inserted fields: %s") % ", ".join(duplicates))
+
+ # Check that field names are not in input
+ if self.input: # Write special version for NewFieldSet?
+ for name in new_names:
+ if name in self.input and name not in self._deleted:
+ raise UniqKeyError(_("Field name '%s' already exists") % name)
+
+ # Check that field names are not in inserted fields
+ for fields in self._inserted.itervalues():
+ for field in fields:
+ if field.name in new_names:
+ raise UniqKeyError(_("Field name '%s' already exists") % field.name)
+
+ # Input have already inserted field?
+ if key in self._inserted:
+ if next:
+ self._inserted[key].extend( reversed(new_fields) )
+ else:
+ self._inserted[key].extendleft( reversed(new_fields) )
+ return
+
+ # Whould like to insert in inserted fields?
+ if key:
+ for fields in self._inserted.itervalues():
+ names = [item.name for item in fields]
+ try:
+ pos = names.index(key)
+ except ValueError:
+ continue
+ if 0 <= pos:
+ if next:
+ pos += 1
+ fields.rotate(-pos)
+ fields.extendleft( reversed(new_fields) )
+ fields.rotate(pos)
+ return
+
+ # Get next field. Use None if we are at the end.
+ if next:
+ index = self.input[key].index + 1
+ try:
+ key = self.input[index].name
+ except IndexError:
+ key = None
+
+ # Check that field names are not in input
+ if key not in self.input:
+ raise MissingField(self, key)
+
+ # Insert in original input
+ self._inserted[key]= deque(new_fields)
+
+ def _getDescription(self):
+ return self.input.description
+ description = property(_getDescription)
+
+ def _getStream(self):
+ # FIXME: This property is maybe a bad idea since address may be differents
+ return self.input.stream
+ stream = property(_getStream)
+
+ def _getName(self):
+ return self.input.name
+ name = property(_getName)
+
+ def _getEndian(self):
+ return self.input.endian
+ endian = property(_getEndian)
+
+ def _getAddress(self):
+ if self._parent:
+ return self._parent._getFieldAddress(self.name)
+ else:
+ return 0
+ address = property(_getAddress)
+
+ def _getAbsoluteAddress(self):
+ address = self.address
+ current = self._parent
+ while current:
+ address += current.address
+ current = current._parent
+ return address
+ absolute_address = property(_getAbsoluteAddress)
+
+ def hasValue(self):
+ return False
+# return self._parent.input[self.name].hasValue()
+
+ def _getSize(self):
+ if self.is_altered:
+ return sum(field.size for field in self)
+ else:
+ return self.input.size
+ size = property(_getSize)
+
+ def _getPath(self):
+ return self.input.path
+ path = property(_getPath)
+
+ def _getOriginalField(self, name):
+ assert name in self.input
+ return self.input[name]
+
+ def _getFieldInputAddress(self, name):
+ """
+ Absolute address of a field from the input field set.
+ """
+ assert name in self.input
+ return self.input[name].absolute_address
+
+ def _getFieldAddress(self, name):
+ """
+ Compute relative address of a field. The operation takes care of
+ deleted and resized fields.
+ """
+ #assert name not in self._deleted
+ addr = 0
+ for field in self:
+ if field.name == name:
+ return addr
+ addr += field.size
+ raise MissingField(self, name)
+
+ def _getItemByPath(self, path):
+ if not path[0]:
+ path = path[1:]
+ field = self
+ for name in path:
+ field = field[name]
+ return field
+
+ def __contains__(self, name):
+ try:
+ field = self[name]
+ return (field is not None)
+ except MissingField:
+ return False
+
+ def __getitem__(self, key):
+ """
+ Create a weak reference to an editable field (EditableField) for the
+ field with specified name. If the field is removed later, using the
+ editable field will raise a weakref.ReferenceError exception.
+
+ May raise a MissingField error if the field doesn't exist in original
+ field set or it has been deleted.
+ """
+ if "/" in key:
+ return self._getItemByPath(key.split("/"))
+ if isinstance(key, (int, long)):
+ raise EditorError("Integer index are not supported")
+
+ if (key in self._deleted) or (key not in self.input):
+ raise MissingField(self, key)
+ if key not in self._fields:
+ field = self.input[key]
+ if field.is_field_set:
+ self._fields[key] = createEditableFieldSet(self, field)
+ else:
+ self._fields[key] = createEditableField(self, field)
+ return weakref.proxy(self._fields[key])
+
+ def __delitem__(self, name):
+ """
+ Remove a field from the field set. May raise an MissingField exception
+ if the field has already been deleted.
+ """
+ parts = name.partition('/')
+ if parts[2]:
+ fieldset = self[parts[0]]
+ del fieldset[parts[2]]
+ return
+ if name in self._deleted:
+ raise MissingField(self, name)
+ self._deleted.add(name)
+ if name in self._fields:
+ del self._fields[name]
+
+ def writeInto(self, output):
+ """
+ Write the content if this field set into the output stream
+ (OutputStream).
+ """
+ if not self.is_altered:
+ # Not altered: just copy bits/bytes
+ input = self.input
+ if input.size % 8:
+ output.copyBitsFrom(input.stream,
+ input.absolute_address, input.size, input.endian)
+ else:
+ output.copyBytesFrom(input.stream,
+ input.absolute_address, input.size//8)
+ else:
+ # Altered: call writeInto() method of each field
+ realaddr = 0
+ for field in self:
+ field.writeInto(output)
+ realaddr += field.size
+
+ def _getValue(self):
+ raise EditorError('Field set "%s" has no value' % self.path)
+ def _setValue(self, value):
+ raise EditorError('Field set "%s" value is read only' % self.path)
+ value = property(_getValue, _setValue, "Value of field")
+
+class EditableFloat(EditableFieldSet):
+ _value = None
+
+ def _isAltered(self):
+ return (self._value is not None)
+ is_altered = property(_isAltered)
+
+ def writeInto(self, output):
+ if self._value is not None:
+ self._write(output)
+ else:
+ EditableFieldSet.writeInto(self, output)
+
+ def _write(self, output):
+ format = self.input.struct_format
+ raw = struct.pack(format, self._value)
+ output.writeBytes(raw)
+
+ def _setValue(self, value):
+ self.parent._is_altered = True
+ self._value = value
+ value = property(EditableFieldSet._getValue, _setValue)
+
+def createEditableFieldSet(parent, field):
+ cls = field.__class__
+ # FIXME: Support Float80
+ if cls in (Float32, Float64):
+ return EditableFloat(parent, field)
+ else:
+ return EditableFieldSet(parent, field)
+
+class NewFieldSet(EditableFieldSet):
+ def __init__(self, parent, name):
+ EditableFieldSet.__init__(self, parent, None)
+ self._name = name
+ self._endian = parent.endian
+
+ def __iter__(self):
+ if None in self._inserted:
+ return iter(self._inserted[None])
+ else:
+ raise StopIteration()
+
+ def _getName(self):
+ return self._name
+ name = property(_getName)
+
+ def _getEndian(self):
+ return self._endian
+ endian = property(_getEndian)
+
+ is_altered = property(lambda self: True)
+
+def createEditor(fieldset):
+ return EditableFieldSet(None, fieldset)
+
diff --git a/MAT/hachoir_editor/typed_field.py b/MAT/hachoir_editor/typed_field.py
new file mode 100644
index 0000000..0f0427b
--- /dev/null
+++ b/MAT/hachoir_editor/typed_field.py
@@ -0,0 +1,253 @@
+from hachoir_core.field import (
+ RawBits, Bit, Bits, PaddingBits,
+ RawBytes, Bytes, PaddingBytes,
+ GenericString, Character,
+ isInteger, isString)
+from field import FakeField
+
+class EditableField(FakeField):
+ """
+ Pure virtual class used to write editable field class.
+ """
+
+ _is_altered = False
+ def __init__(self, parent, name, value=None):
+ FakeField.__init__(self, parent, name)
+ self._value = value
+
+ def _isAltered(self):
+ return self._is_altered
+ is_altered = property(_isAltered)
+
+ def hasValue(self):
+ return True
+
+ def _computeSize(self):
+ raise NotImplementedError()
+ def _getValue(self):
+ return self._value
+ def _setValue(self, value):
+ self._value = value
+
+ def _propGetValue(self):
+ if self._value is not None:
+ return self._getValue()
+ else:
+ return FakeField._getValue(self)
+ def _propSetValue(self, value):
+ self._setValue(value)
+ self._is_altered = True
+ value = property(_propGetValue, _propSetValue)
+
+ def _getSize(self):
+ if self._value is not None:
+ return self._computeSize()
+ else:
+ return FakeField._getSize(self)
+ size = property(_getSize)
+
+ def _write(self, output):
+ raise NotImplementedError()
+
+ def writeInto(self, output):
+ if self._is_altered:
+ self._write(output)
+ else:
+ return FakeField.writeInto(self, output)
+
+class EditableFixedField(EditableField):
+ """
+ Editable field with fixed size.
+ """
+
+ def __init__(self, parent, name, value=None, size=None):
+ EditableField.__init__(self, parent, name, value)
+ if size is not None:
+ self._size = size
+ else:
+ self._size = self._parent._getOriginalField(self._name).size
+
+ def _getSize(self):
+ return self._size
+ size = property(_getSize)
+
+class EditableBits(EditableFixedField):
+ def __init__(self, parent, name, *args):
+ if args:
+ if len(args) != 2:
+ raise TypeError(
+ "Wrong argument count, EditableBits constructor prototype is: "
+ "(parent, name, [size, value])")
+ size = args[0]
+ value = args[1]
+ assert isinstance(value, (int, long))
+ else:
+ size = None
+ value = None
+ EditableFixedField.__init__(self, parent, name, value, size)
+ if args:
+ self._setValue(args[1])
+ self._is_altered = True
+
+ def _setValue(self, value):
+ if not(0 <= value < (1 << self._size)):
+ raise ValueError("Invalid value, must be in range %s..%s"
+ % (0, (1 << self._size) - 1))
+ self._value = value
+
+ def _write(self, output):
+ output.writeBits(self._size, self._value, self._parent.endian)
+
+class EditableBytes(EditableField):
+ def _setValue(self, value):
+ if not value: raise ValueError(
+ "Unable to set empty string to a EditableBytes field")
+ self._value = value
+
+ def _computeSize(self):
+ return len(self._value) * 8
+
+ def _write(self, output):
+ output.writeBytes(self._value)
+
+class EditableString(EditableField):
+ MAX_SIZE = {
+ "Pascal8": (1 << 8)-1,
+ "Pascal16": (1 << 16)-1,
+ "Pascal32": (1 << 32)-1,
+ }
+
+ def __init__(self, parent, name, *args, **kw):
+ if len(args) == 2:
+ value = args[1]
+ assert isinstance(value, str) # TODO: support Unicode
+ elif not args:
+ value = None
+ else:
+ raise TypeError(
+ "Wrong argument count, EditableString constructor prototype is:"
+ "(parent, name, [format, value])")
+ EditableField.__init__(self, parent, name, value)
+ if len(args) == 2:
+ self._charset = kw.get('charset', None)
+ self._format = args[0]
+ if self._format in GenericString.PASCAL_FORMATS:
+ self._prefix_size = GenericString.PASCAL_FORMATS[self._format]
+ else:
+ self._prefix_size = 0
+ self._suffix_str = GenericString.staticSuffixStr(
+ self._format, self._charset, self._parent.endian)
+ self._is_altered = True
+ else:
+ orig = self._parent._getOriginalField(name)
+ self._charset = orig.charset
+ self._format = orig.format
+ self._prefix_size = orig.content_offset
+ self._suffix_str = orig.suffix_str
+
+ def _setValue(self, value):
+ size = len(value)
+ if self._format in self.MAX_SIZE and self.MAX_SIZE[self._format] < size:
+ raise ValueError("String is too big")
+ self._value = value
+
+ def _computeSize(self):
+ return (self._prefix_size + len(self._value) + len(self._suffix_str))*8
+
+ def _write(self, output):
+ if self._format in GenericString.SUFFIX_FORMAT:
+ output.writeBytes(self._value)
+ output.writeBytes(self._suffix_str)
+ elif self._format == "fixed":
+ output.writeBytes(self._value)
+ else:
+ assert self._format in GenericString.PASCAL_FORMATS
+ size = GenericString.PASCAL_FORMATS[self._format]
+ output.writeInteger(len(self._value), False, size, self._parent.endian)
+ output.writeBytes(self._value)
+
+class EditableCharacter(EditableFixedField):
+ def __init__(self, parent, name, *args):
+ if args:
+ if len(args) != 3:
+ raise TypeError(
+ "Wrong argument count, EditableCharacter "
+ "constructor prototype is: (parent, name, [value])")
+ value = args[0]
+ if not isinstance(value, str) or len(value) != 1:
+ raise TypeError("EditableCharacter needs a character")
+ else:
+ value = None
+ EditableFixedField.__init__(self, parent, name, value, 8)
+ if args:
+ self._is_altered = True
+
+ def _setValue(self, value):
+ if not isinstance(value, str) or len(value) != 1:
+ raise TypeError("EditableCharacter needs a character")
+ self._value = value
+
+ def _write(self, output):
+ output.writeBytes(self._value)
+
+class EditableInteger(EditableFixedField):
+ VALID_VALUE_SIGNED = {
+ 8: (-(1 << 8), (1 << 8)-1),
+ 16: (-(1 << 15), (1 << 15)-1),
+ 32: (-(1 << 31), (1 << 31)-1),
+ }
+ VALID_VALUE_UNSIGNED = {
+ 8: (0, (1 << 8)-1),
+ 16: (0, (1 << 16)-1),
+ 32: (0, (1 << 32)-1)
+ }
+
+ def __init__(self, parent, name, *args):
+ if args:
+ if len(args) != 3:
+ raise TypeError(
+ "Wrong argument count, EditableInteger constructor prototype is: "
+ "(parent, name, [signed, size, value])")
+ size = args[1]
+ value = args[2]
+ assert isinstance(value, (int, long))
+ else:
+ size = None
+ value = None
+ EditableFixedField.__init__(self, parent, name, value, size)
+ if args:
+ self._signed = args[0]
+ self._is_altered = True
+ else:
+ self._signed = self._parent._getOriginalField(self._name).signed
+
+ def _setValue(self, value):
+ if self._signed:
+ valid = self.VALID_VALUE_SIGNED
+ else:
+ valid = self.VALID_VALUE_UNSIGNED
+ minval, maxval = valid[self._size]
+ if not(minval <= value <= maxval):
+ raise ValueError("Invalid value, must be in range %s..%s"
+ % (minval, maxval))
+ self._value = value
+
+ def _write(self, output):
+ output.writeInteger(
+ self.value, self._signed, self._size//8, self._parent.endian)
+
+def createEditableField(fieldset, field):
+ if isInteger(field):
+ cls = EditableInteger
+ elif isString(field):
+ cls = EditableString
+ elif field.__class__ in (RawBytes, Bytes, PaddingBytes):
+ cls = EditableBytes
+ elif field.__class__ in (RawBits, Bits, Bit, PaddingBits):
+ cls = EditableBits
+ elif field.__class__ == Character:
+ cls = EditableCharacter
+ else:
+ cls = FakeField
+ return cls(fieldset, field.name)
+
diff --git a/MAT/images.py b/MAT/images.py
new file mode 100644
index 0000000..236d566
--- /dev/null
+++ b/MAT/images.py
@@ -0,0 +1,41 @@
+'''
+ Takes care about pictures formats
+'''
+
+import parser
+
+
+class JpegStripper(parser.GenericParser):
+ '''
+ represents a jpeg file
+ '''
+ def _should_remove(self, field):
+ '''
+ return True if the field is compromizing
+ '''
+ field_list = frozenset(['start_image', 'app0', 'start_frame',
+ 'start_scan', 'data', 'end_image'])
+ if field.name in field_list:
+ return False
+ elif field.name.startswith('quantization['):
+ return False
+ elif field.name.startswith('huffman['):
+ return False
+ return True
+
+
+class PngStripper(parser.GenericParser):
+ '''
+ represents a png file
+ see : http://www.sno.phy.queensu.ca/~phil/exiftool/TagNames/PNG.html
+ '''
+ def _should_remove(self, field):
+ '''
+ return True if the field is compromizing
+ '''
+ field_list = frozenset(['id', 'header', 'physical', 'end'])
+ if field.name in field_list:
+ return False
+ if field.name.startswith('data['):
+ return False
+ return True
diff --git a/MAT/mat.py b/MAT/mat.py
new file mode 100644
index 0000000..4c12333
--- /dev/null
+++ b/MAT/mat.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+
+'''
+ Metadata anonymisation toolkit library
+'''
+
+import os
+import subprocess
+import logging
+import mimetypes
+import xml.sax
+
+import hachoir_core.cmd_line
+import hachoir_parser
+
+import strippers
+
+__version__ = '0.3.2'
+__author__ = 'jvoisin'
+
+#Silence
+LOGGING_LEVEL = logging.CRITICAL
+hachoir_core.config.quiet = True
+fname = ''
+
+#Verbose
+LOGGING_LEVEL = logging.DEBUG
+#hachoir_core.config.quiet = False
+#logname = 'report.log'
+
+logging.basicConfig(filename=fname, level=LOGGING_LEVEL)
+
+
+def get_sharedir(filename):
+ '''
+ An ugly hack to find various files
+ '''
+ if os.path.isfile(filename):
+ return filename
+ elif os.path.exists(os.path.join('/usr/local/share/mat/', filename)):
+ return os.path.join('/usr/local/share/mat/', filename)
+ elif os.path.exists(os.path.join('/usr/share/mat/', filename)):
+ return os.path.join('/usr/share/mat', filename)
+ elif os.path.exists(os.path.join('/usr/local/share/pixmaps/', filename)):
+ return os.path.join('/usr/local/share/pixmaps/', filename)
+
+
+class XMLParser(xml.sax.handler.ContentHandler):
+ '''
+ Parse the supported format xml, and return a corresponding
+ list of dict
+ '''
+ def __init__(self):
+ self.dict = {}
+ self.list = []
+ self.content, self.key = '', ''
+ self.between = False
+
+ def startElement(self, name, attrs):
+ '''
+ Called when entering into xml balise
+ '''
+ self.between = True
+ self.key = name
+ self.content = ''
+
+ def endElement(self, name):
+ '''
+ Called when exiting a xml balise
+ '''
+ if name == 'format': # exiting a fileformat section
+ self.list.append(self.dict.copy())
+ self.dict.clear()
+ else:
+ content = self.content.replace('\s', ' ')
+ self.dict[self.key] = content
+ self.between = False
+
+ def characters(self, characters):
+ '''
+ Concatenate the content between opening and closing balises
+ '''
+ if self.between:
+ self.content += characters
+
+
+def secure_remove(filename):
+ '''
+ securely remove the file
+ '''
+ removed = False
+ try:
+ subprocess.call(['shred', '--remove', filename])
+ removed = True
+ except:
+ logging.error('Unable to securely remove %s' % filename)
+
+ if removed is False:
+ try:
+ os.remove(filename)
+ except:
+ logging.error('Unable to remove %s' % filename)
+
+
+def create_class_file(name, backup, add2archive):
+ '''
+ return a $FILETYPEStripper() class,
+ corresponding to the filetype of the given file
+ '''
+ if not os.path.isfile(name):
+ # check if the file exists
+ logging.error('%s is not a valid file' % name)
+ return None
+
+ if not os.access(name, os.R_OK):
+ #check read permissions
+ logging.error('%s is is not readable' % name)
+ return None
+
+ if not os.access(name, os.W_OK):
+ #check write permission
+ logging.error('%s is not writtable' % name)
+ return None
+
+ filename = ''
+ try:
+ filename = hachoir_core.cmd_line.unicodeFilename(name)
+ except TypeError: # get rid of "decoding Unicode is not supported"
+ filename = name
+
+ parser = hachoir_parser.createParser(filename)
+ if not parser:
+ logging.info('Unable to parse %s' % filename)
+ return None
+
+ mime = parser.mime_type
+
+ if mime == 'application/zip': # some formats are zipped stuff
+ mime = mimetypes.guess_type(name)[0]
+
+ if mime.startswith('application/vnd.oasis.opendocument'):
+ mime = 'application/opendocument' # opendocument fileformat
+ elif mime.startswith('application/vnd.openxmlformats-officedocument'):
+ mime = 'application/officeopenxml' # office openxml
+
+ try:
+ stripper_class = strippers.STRIPPERS[mime]
+ except KeyError:
+ logging.info('Don\'t have stripper for %s format' % mime)
+ return None
+
+ return stripper_class(filename, parser, mime, backup, add2archive)
diff --git a/MAT/misc.py b/MAT/misc.py
new file mode 100644
index 0000000..d084861
--- /dev/null
+++ b/MAT/misc.py
@@ -0,0 +1,63 @@
+'''
+ Care about misc formats
+'''
+
+import parser
+
+from bencode import bencode
+
+
+class TorrentStripper(parser.GenericParser):
+ '''
+ Represent a torrent file with the help
+ of the bencode lib from Petru Paler
+ '''
+ def __init__(self, filename, parser, mime, backup, add2archive):
+ super(TorrentStripper, self).__init__(filename, parser, mime,
+ backup, add2archive)
+ self.fields = ['comment', 'creation date', 'created by']
+
+ def is_clean(self):
+ '''
+ Check if the file is clean from harmful metadatas
+ '''
+ with open(self.filename, 'r') as f:
+ decoded = bencode.bdecode(f.read())
+ for key in self.fields:
+ try:
+ if decoded[key] != '':
+ return False
+ except:
+ pass
+ return True
+
+ def get_meta(self):
+ '''
+ Return a dict with all the meta of the file
+ '''
+ metadata = {}
+ with open(self.filename, 'r') as f:
+ decoded = bencode.bdecode(f.read())
+ for key in self.fields:
+ try:
+ if decoded[key] != '':
+ metadata[key] = decoded[key]
+ except:
+ pass
+ return metadata
+
+ def remove_all(self):
+ '''
+ Remove all the files that are compromizing
+ '''
+ with open(self.filename, 'r') as f:
+ decoded = bencode.bdecode(f.read())
+ for key in self.fields:
+ try:
+ decoded[key] = ''
+ except:
+ pass
+ with open(self.output, 'w') as f: # encode the decoded torrent
+ f.write(bencode.bencode(decoded)) # and write it in self.output
+ self.do_backup()
+ return True
diff --git a/MAT/office.py b/MAT/office.py
new file mode 100644
index 0000000..d14125b
--- /dev/null
+++ b/MAT/office.py
@@ -0,0 +1,265 @@
+'''
+ Care about office's formats
+'''
+
+import os
+import logging
+import zipfile
+import fileinput
+import xml.dom.minidom as minidom
+
+try:
+ import cairo
+ import poppler
+except ImportError:
+ pass
+
+import mat
+import parser
+import archive
+
+
+class OpenDocumentStripper(archive.GenericArchiveStripper):
+ '''
+ An open document file is a zip, with xml file into.
+ The one that interest us is meta.xml
+ '''
+
+ def get_meta(self):
+ '''
+ Return a dict with all the meta of the file by
+ trying to read the meta.xml file.
+ '''
+ zipin = zipfile.ZipFile(self.filename, 'r')
+ metadata = {}
+ try:
+ content = zipin.read('meta.xml')
+ dom1 = minidom.parseString(content)
+ elements = dom1.getElementsByTagName('office:meta')
+ for i in elements[0].childNodes:
+ if i.tagName != 'meta:document-statistic':
+ nodename = ''.join([k for k in i.nodeName.split(':')[1:]])
+ metadata[nodename] = ''.join([j.data for j in i.childNodes])
+ else:
+ # thank you w3c for not providing a nice
+ # method to get all attributes from a node
+ pass
+ zipin.close()
+ except KeyError: # no meta.xml file found
+ logging.debug('%s has no opendocument metadata' % self.filename)
+ return metadata
+
+ def _remove_all(self):
+ '''
+ FIXME ?
+ There is a patch implementing the Zipfile.remove()
+ method here : http://bugs.python.org/issue6818
+ '''
+ zipin = zipfile.ZipFile(self.filename, 'r')
+ zipout = zipfile.ZipFile(self.output, 'w', allowZip64=True)
+
+ for item in zipin.namelist():
+ name = os.path.join(self.tempdir, item)
+ _, ext = os.path.splitext(name)
+
+ if item.endswith('manifest.xml'):
+ # contain the list of all files present in the archive
+ zipin.extract(item, self.tempdir)
+ for line in fileinput.input(name, inplace=1):
+ #remove the line which contains "meta.xml"
+ line = line.strip()
+ if not 'meta.xml' in line:
+ print line
+ zipout.write(name, item)
+
+ elif ext in parser.NOMETA or item == 'mimetype':
+ #keep NOMETA files, and the "manifest" file
+ if item != 'meta.xml': # contains the metadata
+ zipin.extract(item, self.tempdir)
+ zipout.write(name, item)
+
+ else:
+ zipin.extract(item, self.tempdir)
+ if os.path.isfile(name):
+ try:
+ cfile = mat.create_class_file(name, False,
+ self.add2archive)
+ cfile.remove_all()
+ logging.debug('Processing %s from %s' % (item,
+ self.filename))
+ zipout.write(name, item)
+ except:
+ logging.info('%s\' fileformat is not supported' % item)
+ if self.add2archive:
+ zipout.write(name, item)
+ zipout.comment = ''
+ logging.info('%s treated' % self.filename)
+ zipin.close()
+ zipout.close()
+ self.do_backup()
+ return True
+
+ def is_clean(self):
+ '''
+ Check if the file is clean from harmful metadatas
+ '''
+ zipin = zipfile.ZipFile(self.filename, 'r')
+ try:
+ zipin.getinfo('meta.xml')
+ except KeyError: # no meta.xml in the file
+ czf = archive.ZipStripper(self.filename, self.parser,
+ 'application/zip', self.backup, self.add2archive)
+ if czf.is_clean():
+ zipin.close()
+ return True
+ zipin.close()
+ return False
+
+
+class PdfStripper(parser.GenericParser):
+ '''
+ Represent a PDF file
+ '''
+ def __init__(self, filename, parser, mime, backup, add2archive):
+ super(PdfStripper, self).__init__(filename, parser, mime, backup,
+ add2archive)
+ uri = 'file://' + os.path.abspath(self.filename)
+ self.password = None
+ self.document = poppler.document_new_from_file(uri, self.password)
+ self.meta_list = frozenset(['title', 'author', 'subject', 'keywords', 'creator',
+ 'producer', 'metadata'])
+
+ def is_clean(self):
+ '''
+ Check if the file is clean from harmful metadatas
+ '''
+ for key in self.meta_list:
+ if self.document.get_property(key):
+ return False
+ return True
+
+ def remove_all(self):
+ '''
+ Remove metadata
+ '''
+ return self._remove_meta()
+
+ def _remove_meta(self):
+ '''
+ Opening the PDF with poppler, then doing a render
+ on a cairo pdfsurface for each pages.
+
+ http://cairographics.org/documentation/pycairo/2/
+ python-poppler is not documented at all : have fun ;)
+ '''
+ page = self.document.get_page(0)
+ # assume that every pages are the same size
+ page_width, page_height = page.get_size()
+ surface = cairo.PDFSurface(self.output, page_width, page_height)
+ context = cairo.Context(surface) # context draws on the surface
+ logging.debug('PDF rendering of %s' % self.filename)
+ for pagenum in xrange(self.document.get_n_pages()):
+ page = self.document.get_page(pagenum)
+ context.translate(0, 0)
+ page.render_for_printing(context) # render the page on context
+ context.show_page() # draw context on surface
+ surface.finish()
+
+ try:
+ import pdfrw # For now, poppler cannot write meta, so we must use pdfrw
+ logging.debug('Removing %s\'s superficial metadata' % self.filename)
+ trailer = pdfrw.PdfReader(self.output)
+ trailer.Info.Producer = None
+ trailer.Info.Creator = None
+ writer = pdfrw.PdfWriter()
+ writer.trailer = trailer
+ writer.write(self.output)
+ self.do_backup()
+ return True
+ except:
+ print('Unable to remove all metadata from %s, please install\
+pdfrw' % self.output)
+ return False
+ return True
+
+ def get_meta(self):
+ '''
+ Return a dict with all the meta of the file
+ '''
+ metadata = {}
+ for key in self.meta_list:
+ if self.document.get_property(key):
+ metadata[key] = self.document.get_property(key)
+ return metadata
+
+
+class OpenXmlStripper(archive.GenericArchiveStripper):
+ '''
+ Represent an office openxml document, which is like
+ an opendocument format, with some tricky stuff added.
+ It contains mostly xml, but can have media blobs, crap, ...
+ (I don't like this format.)
+ '''
+ def _remove_all(self):
+ '''
+ FIXME ?
+ There is a patch implementing the Zipfile.remove()
+ method here : http://bugs.python.org/issue6818
+ '''
+ zipin = zipfile.ZipFile(self.filename, 'r')
+ zipout = zipfile.ZipFile(self.output, 'w',
+ allowZip64=True)
+ for item in zipin.namelist():
+ name = os.path.join(self.tempdir, item)
+ _, ext = os.path.splitext(name)
+ if item.startswith('docProps/'): # metadatas
+ pass
+ elif ext in parser.NOMETA or item == '.rels':
+ #keep parser.NOMETA files, and the file named ".rels"
+ zipin.extract(item, self.tempdir)
+ zipout.write(name, item)
+ else:
+ zipin.extract(item, self.tempdir)
+ if os.path.isfile(name): # don't care about folders
+ try:
+ cfile = mat.create_class_file(name, False,
+ self.add2archive)
+ cfile.remove_all()
+ logging.debug('Processing %s from %s' % (item,
+ self.filename))
+ zipout.write(name, item)
+ except:
+ logging.info('%s\' fileformat is not supported' % item)
+ if self.add2archive:
+ zipout.write(name, item)
+ zipout.comment = ''
+ logging.info('%s treated' % self.filename)
+ zipin.close()
+ zipout.close()
+ self.do_backup()
+ return True
+
+ def is_clean(self):
+ '''
+ Check if the file is clean from harmful metadatas
+ '''
+ zipin = zipfile.ZipFile(self.filename, 'r')
+ for item in zipin.namelist():
+ if item.startswith('docProps/'):
+ return False
+ zipin.close()
+ czf = archive.ZipStripper(self.filename, self.parser,
+ 'application/zip', self.backup, self.add2archive)
+ return czf.is_clean()
+
+ def get_meta(self):
+ '''
+ Return a dict with all the meta of the file
+ '''
+ zipin = zipfile.ZipFile(self.filename, 'r')
+ metadata = {}
+ for item in zipin.namelist():
+ if item.startswith('docProps/'):
+ metadata[item] = 'harmful content'
+ zipin.close()
+ return metadata
diff --git a/MAT/parser.py b/MAT/parser.py
new file mode 100644
index 0000000..d2eaf9c
--- /dev/null
+++ b/MAT/parser.py
@@ -0,0 +1,120 @@
+'''
+ Parent class of all parser
+'''
+
+import hachoir_core
+import hachoir_editor
+
+import os
+
+import mat
+
+NOMETA = ('.bmp', '.rdf', '.txt', '.xml', '.rels')
+#bmp : image
+#rdf : text
+#txt : plain text
+#xml : formated text
+#rels : openxml foramted text
+
+
+FIELD = object()
+
+class GenericParser(object):
+ '''
+ Parent class of all parsers
+ '''
+ def __init__(self, filename, parser, mime, backup, add2archive):
+ self.filename = ''
+ self.parser = parser
+ self.mime = mime
+ self.backup = backup
+ self.editor = hachoir_editor.createEditor(parser)
+ self.realname = filename
+ try:
+ self.filename = hachoir_core.cmd_line.unicodeFilename(filename)
+ except TypeError: # get rid of "decoding Unicode is not supported"
+ self.filename = filename
+ basename, ext = os.path.splitext(filename)
+ self.output = basename + '.cleaned' + ext
+ self.basename = os.path.basename(filename) # only filename
+
+ def is_clean(self):
+ '''
+ Check if the file is clean from harmful metadatas
+ '''
+ for field in self.editor:
+ if self._should_remove(field):
+ return self._is_clean(self.editor)
+ return True
+
+ def _is_clean(self, fieldset):
+ for field in fieldset:
+ remove = self._should_remove(field)
+ if remove is True:
+ return False
+ if remove is FIELD:
+ if not self._is_clean(field):
+ return False
+ return True
+
+ def remove_all(self):
+ '''
+ Remove all the files that are compromizing
+ '''
+ state = self._remove_all(self.editor)
+ hachoir_core.field.writeIntoFile(self.editor, self.output)
+ self.do_backup()
+ return state
+
+ def _remove_all(self, fieldset):
+ try:
+ for field in fieldset:
+ remove = self._should_remove(field)
+ if remove is True:
+ self._remove(fieldset, field.name)
+ if remove is FIELD:
+ self._remove_all(field)
+ return True
+ except:
+ return False
+
+ def _remove(self, fieldset, field):
+ '''
+ Delete the given field
+ '''
+ del fieldset[field]
+
+ def get_meta(self):
+ '''
+ Return a dict with all the meta of the file
+ '''
+ metadata = {}
+ self._get_meta(self.editor, metadata)
+ return metadata
+
+ def _get_meta(self, fieldset, metadata):
+ for field in fieldset:
+ remove = self._should_remove(field)
+ if remove is True:
+ try:
+ metadata[field.name] = field.value
+ except:
+ metadata[field.name] = 'harmful content'
+ if remove is FIELD:
+ self._get_meta(field)
+
+ def _should_remove(self, key):
+ '''
+ return True if the field is compromizing
+ abstract method
+ '''
+ raise NotImplementedError
+
+ def do_backup(self):
+ '''
+ Do a backup of the file if asked,
+ and change his creation/access date
+ '''
+ if self.backup is False:
+ mat.secure_remove(self.filename)
+ os.rename(self.output, self.filename)
diff --git a/MAT/strippers.py b/MAT/strippers.py
new file mode 100644
index 0000000..61030a7
--- /dev/null
+++ b/MAT/strippers.py
@@ -0,0 +1,67 @@
+'''
+ Manage which fileformat can be processed
+'''
+
+import images
+import audio
+import office
+import archive
+import misc
+import subprocess
+
+STRIPPERS = {
+ 'application/x-tar': archive.TarStripper,
+ 'application/x-gzip': archive.GzipStripper,
+ 'application/x-bzip2': archive.Bzip2Stripper,
+ 'application/zip': archive.ZipStripper,
+ 'audio/mpeg': audio.MpegAudioStripper,
+ 'application/x-bittorrent': misc.TorrentStripper,
+ 'application/opendocument': office.OpenDocumentStripper,
+ 'application/officeopenxml': office.OpenXmlStripper,
+}
+
+
+# PDF support
+pdfSupport = True
+try:
+ import poppler
+except ImportError:
+ print('Unable to import python-poppler: not PDF support')
+ pdfSupport = False
+
+try:
+ import cairo
+except ImportError:
+ print('Unable to import python-cairo: no PDF support')
+ pdfSupport = False
+
+try:
+ import pdfrw
+except ImportError:
+ print('Unable to import python-pdfrw: no PDf support')
+ pdfSupport = False
+
+if pdfSupport:
+ STRIPPERS['application/x-pdf'] = office.PdfStripper
+ STRIPPERS['application/pdf'] = office.PdfStripper
+
+
+# audio format support with mutagen-python
+try:
+ import mutagen
+ STRIPPERS['audio/x-flac'] = audio.FlacStripper
+ STRIPPERS['audio/vorbis'] = audio.OggStripper
+except ImportError:
+ print('Unable to import python-mutagen: limited audio format support')
+
+# exiftool
+try:
+ subprocess.Popen('exiftool', stdout=open('/dev/null'))
+ import exiftool
+ STRIPPERS['image/jpeg'] = exiftool.JpegStripper
+ STRIPPERS['image/png'] = exiftool.PngStripper
+except: # if exiftool is not installed, use hachoir
+ print('Unable to find exiftool: limited images support')
+ STRIPPERS['image/jpeg'] = images.JpegStripper
+ STRIPPERS['image/png'] = images.PngStripper
+
diff --git a/MAT/tarfile/__init__.py b/MAT/tarfile/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/MAT/tarfile/__init__.py
@@ -0,0 +1 @@
+
diff --git a/MAT/tarfile/tarfile.py b/MAT/tarfile/tarfile.py
new file mode 100644
index 0000000..6978eb1
--- /dev/null
+++ b/MAT/tarfile/tarfile.py
@@ -0,0 +1,2593 @@
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------
+# tarfile.py
+#-------------------------------------------------------------------
+# Copyright (C) 2002 Lars Gustäbel
+# All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+"""Read from and write to tar format archives.
+"""
+
+__version__ = "$Revision$"
+# $Source$
+
+version = "0.9.0"
+__author__ = "Lars Gustäbel (lars@gustaebel.de)"
+__date__ = "$Date$"
+__cvsid__ = "$Id$"
+__credits__ = "Gustavo Niemeyer, Niels Gustäbel, Richard Townsend."
+
+#---------
+# Imports
+#---------
+import sys
+import os
+import shutil
+import stat
+import errno
+import time
+import struct
+import copy
+import re
+import operator
+
+try:
+ import grp, pwd
+except ImportError:
+ grp = pwd = None
+
+# from tarfile import *
+__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"]
+
+#---------------------------------------------------------
+# tar constants
+#---------------------------------------------------------
+NUL = "\0" # the null character
+BLOCKSIZE = 512 # length of processing blocks
+RECORDSIZE = BLOCKSIZE * 20 # length of records
+GNU_MAGIC = "ustar \0" # magic gnu tar string
+POSIX_MAGIC = "ustar\x0000" # magic posix tar string
+
+LENGTH_NAME = 100 # maximum length of a filename
+LENGTH_LINK = 100 # maximum length of a linkname
+LENGTH_PREFIX = 155 # maximum length of the prefix field
+
+REGTYPE = "0" # regular file
+AREGTYPE = "\0" # regular file
+LNKTYPE = "1" # link (inside tarfile)
+SYMTYPE = "2" # symbolic link
+CHRTYPE = "3" # character special device
+BLKTYPE = "4" # block special device
+DIRTYPE = "5" # directory
+FIFOTYPE = "6" # fifo special device
+CONTTYPE = "7" # contiguous file
+
+GNUTYPE_LONGNAME = "L" # GNU tar longname
+GNUTYPE_LONGLINK = "K" # GNU tar longlink
+GNUTYPE_SPARSE = "S" # GNU tar sparse file
+
+XHDTYPE = "x" # POSIX.1-2001 extended header
+XGLTYPE = "g" # POSIX.1-2001 global header
+SOLARIS_XHDTYPE = "X" # Solaris extended header
+
+USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format
+GNU_FORMAT = 1 # GNU tar format
+PAX_FORMAT = 2 # POSIX.1-2001 (pax) format
+DEFAULT_FORMAT = GNU_FORMAT
+
+#---------------------------------------------------------
+# tarfile constants
+#---------------------------------------------------------
+# File types that tarfile supports:
+SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE,
+ SYMTYPE, DIRTYPE, FIFOTYPE,
+ CONTTYPE, CHRTYPE, BLKTYPE,
+ GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
+ GNUTYPE_SPARSE)
+
+# File types that will be treated as a regular file.
+REGULAR_TYPES = (REGTYPE, AREGTYPE,
+ CONTTYPE, GNUTYPE_SPARSE)
+
+# File types that are part of the GNU tar format.
+GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
+ GNUTYPE_SPARSE)
+
+# Fields from a pax header that override a TarInfo attribute.
+PAX_FIELDS = ("path", "linkpath", "size", "mtime",
+ "uid", "gid", "uname", "gname")
+
+# Fields in a pax header that are numbers, all other fields
+# are treated as strings.
+PAX_NUMBER_FIELDS = {
+ "atime": float,
+ "ctime": float,
+ "mtime": float,
+ "uid": int,
+ "gid": int,
+ "size": int
+}
+
+#---------------------------------------------------------
+# Bits used in the mode field, values in octal.
+#---------------------------------------------------------
+S_IFLNK = 0120000 # symbolic link
+S_IFREG = 0100000 # regular file
+S_IFBLK = 0060000 # block device
+S_IFDIR = 0040000 # directory
+S_IFCHR = 0020000 # character device
+S_IFIFO = 0010000 # fifo
+
+TSUID = 04000 # set UID on execution
+TSGID = 02000 # set GID on execution
+TSVTX = 01000 # reserved
+
+TUREAD = 0400 # read by owner
+TUWRITE = 0200 # write by owner
+TUEXEC = 0100 # execute/search by owner
+TGREAD = 0040 # read by group
+TGWRITE = 0020 # write by group
+TGEXEC = 0010 # execute/search by group
+TOREAD = 0004 # read by other
+TOWRITE = 0002 # write by other
+TOEXEC = 0001 # execute/search by other
+
+#---------------------------------------------------------
+# initialization
+#---------------------------------------------------------
+ENCODING = sys.getfilesystemencoding()
+if ENCODING is None:
+ ENCODING = sys.getdefaultencoding()
+
+#---------------------------------------------------------
+# Some useful functions
+#---------------------------------------------------------
+
+def stn(s, length):
+ """Convert a python string to a null-terminated string buffer.
+ """
+ return s[:length] + (length - len(s)) * NUL
+
+def nts(s):
+ """Convert a null-terminated string field to a python string.
+ """
+ # Use the string up to the first null char.
+ p = s.find("\0")
+ if p == -1:
+ return s
+ return s[:p]
+
+def nti(s):
+ """Convert a number field to a python number.
+ """
+ # There are two possible encodings for a number field, see
+ # itn() below.
+ if s[0] != chr(0200):
+ try:
+ n = int(nts(s) or "0", 8)
+ except ValueError:
+ raise InvalidHeaderError("invalid header")
+ else:
+ n = 0L
+ for i in xrange(len(s) - 1):
+ n <<= 8
+ n += ord(s[i + 1])
+ return n
+
+def itn(n, digits=8, format=DEFAULT_FORMAT):
+ """Convert a python number to a number field.
+ """
+ # POSIX 1003.1-1988 requires numbers to be encoded as a string of
+ # octal digits followed by a null-byte, this allows values up to
+ # (8**(digits-1))-1. GNU tar allows storing numbers greater than
+ # that if necessary. A leading 0200 byte indicates this particular
+ # encoding, the following digits-1 bytes are a big-endian
+ # representation. This allows values up to (256**(digits-1))-1.
+ if 0 <= n < 8 ** (digits - 1):
+ s = "%0*o" % (digits - 1, n) + NUL
+ else:
+ if format != GNU_FORMAT or n >= 256 ** (digits - 1):
+ raise ValueError("overflow in number field")
+
+ if n < 0:
+ # XXX We mimic GNU tar's behaviour with negative numbers,
+ # this could raise OverflowError.
+ n = struct.unpack("L", struct.pack("l", n))[0]
+
+ s = ""
+ for i in xrange(digits - 1):
+ s = chr(n & 0377) + s
+ n >>= 8
+ s = chr(0200) + s
+ return s
+
+def uts(s, encoding, errors):
+ """Convert a unicode object to a string.
+ """
+ if errors == "utf-8":
+ # An extra error handler similar to the -o invalid=UTF-8 option
+ # in POSIX.1-2001. Replace untranslatable characters with their
+ # UTF-8 representation.
+ try:
+ return s.encode(encoding, "strict")
+ except UnicodeEncodeError:
+ x = []
+ for c in s:
+ try:
+ x.append(c.encode(encoding, "strict"))
+ except UnicodeEncodeError:
+ x.append(c.encode("utf8"))
+ return "".join(x)
+ else:
+ return s.encode(encoding, errors)
+
+def calc_chksums(buf):
+ """Calculate the checksum for a member's header by summing up all
+ characters except for the chksum field which is treated as if
+ it was filled with spaces. According to the GNU tar sources,
+ some tars (Sun and NeXT) calculate chksum with signed char,
+ which will be different if there are chars in the buffer with
+ the high bit set. So we calculate two checksums, unsigned and
+ signed.
+ """
+ unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
+ signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
+ return unsigned_chksum, signed_chksum
+
+def copyfileobj(src, dst, length=None):
+ """Copy length bytes from fileobj src to fileobj dst.
+ If length is None, copy the entire content.
+ """
+ if length == 0:
+ return
+ if length is None:
+ shutil.copyfileobj(src, dst)
+ return
+
+ BUFSIZE = 16 * 1024
+ blocks, remainder = divmod(length, BUFSIZE)
+ for b in xrange(blocks):
+ buf = src.read(BUFSIZE)
+ if len(buf) < BUFSIZE:
+ raise IOError("end of file reached")
+ dst.write(buf)
+
+ if remainder != 0:
+ buf = src.read(remainder)
+ if len(buf) < remainder:
+ raise IOError("end of file reached")
+ dst.write(buf)
+ return
+
+filemode_table = (
+ ((S_IFLNK, "l"),
+ (S_IFREG, "-"),
+ (S_IFBLK, "b"),
+ (S_IFDIR, "d"),
+ (S_IFCHR, "c"),
+ (S_IFIFO, "p")),
+
+ ((TUREAD, "r"),),
+ ((TUWRITE, "w"),),
+ ((TUEXEC|TSUID, "s"),
+ (TSUID, "S"),
+ (TUEXEC, "x")),
+
+ ((TGREAD, "r"),),
+ ((TGWRITE, "w"),),
+ ((TGEXEC|TSGID, "s"),
+ (TSGID, "S"),
+ (TGEXEC, "x")),
+
+ ((TOREAD, "r"),),
+ ((TOWRITE, "w"),),
+ ((TOEXEC|TSVTX, "t"),
+ (TSVTX, "T"),
+ (TOEXEC, "x"))
+)
+
+def filemode(mode):
+ """Convert a file's mode to a string of the form
+ -rwxrwxrwx.
+ Used by TarFile.list()
+ """
+ perm = []
+ for table in filemode_table:
+ for bit, char in table:
+ if mode & bit == bit:
+ perm.append(char)
+ break
+ else:
+ perm.append("-")
+ return "".join(perm)
+
+class TarError(Exception):
+ """Base exception."""
+ pass
+class ExtractError(TarError):
+ """General exception for extract errors."""
+ pass
+class ReadError(TarError):
+ """Exception for unreadble tar archives."""
+ pass
+class CompressionError(TarError):
+ """Exception for unavailable compression methods."""
+ pass
+class StreamError(TarError):
+ """Exception for unsupported operations on stream-like TarFiles."""
+ pass
+class HeaderError(TarError):
+ """Base exception for header errors."""
+ pass
+class EmptyHeaderError(HeaderError):
+ """Exception for empty headers."""
+ pass
+class TruncatedHeaderError(HeaderError):
+ """Exception for truncated headers."""
+ pass
+class EOFHeaderError(HeaderError):
+ """Exception for end of file headers."""
+ pass
+class InvalidHeaderError(HeaderError):
+ """Exception for invalid headers."""
+ pass
+class SubsequentHeaderError(HeaderError):
+ """Exception for missing and invalid extended headers."""
+ pass
+
+#---------------------------
+# internal stream interface
+#---------------------------
+class _LowLevelFile:
+ """Low-level file object. Supports reading and writing.
+ It is used instead of a regular file object for streaming
+ access.
+ """
+
+ def __init__(self, name, mode):
+ mode = {
+ "r": os.O_RDONLY,
+ "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
+ }[mode]
+ if hasattr(os, "O_BINARY"):
+ mode |= os.O_BINARY
+ self.fd = os.open(name, mode, 0666)
+
+ def close(self):
+ os.close(self.fd)
+
+ def read(self, size):
+ return os.read(self.fd, size)
+
+ def write(self, s):
+ os.write(self.fd, s)
+
+class _Stream:
+ """Class that serves as an adapter between TarFile and
+ a stream-like object. The stream-like object only
+ needs to have a read() or write() method and is accessed
+ blockwise. Use of gzip or bzip2 compression is possible.
+ A stream-like object could be for example: sys.stdin,
+ sys.stdout, a socket, a tape device etc.
+
+ _Stream is intended to be used only internally.
+ """
+
+ def __init__(self, name, mode, comptype, fileobj, bufsize):
+ """Construct a _Stream object.
+ """
+ self._extfileobj = True
+ if fileobj is None:
+ fileobj = _LowLevelFile(name, mode)
+ self._extfileobj = False
+
+ if comptype == '*':
+ # Enable transparent compression detection for the
+ # stream interface
+ fileobj = _StreamProxy(fileobj)
+ comptype = fileobj.getcomptype()
+
+ self.name = name or ""
+ self.mode = mode
+ self.comptype = comptype
+ self.fileobj = fileobj
+ self.bufsize = bufsize
+ self.buf = ""
+ self.pos = 0L
+ self.closed = False
+
+ if comptype == "gz":
+ try:
+ import zlib
+ except ImportError:
+ raise CompressionError("zlib module is not available")
+ self.zlib = zlib
+ self.crc = zlib.crc32("") & 0xffffffffL
+ if mode == "r":
+ self._init_read_gz()
+ else:
+ self._init_write_gz()
+
+ if comptype == "bz2":
+ try:
+ import bz2
+ except ImportError:
+ raise CompressionError("bz2 module is not available")
+ if mode == "r":
+ self.dbuf = ""
+ self.cmp = bz2.BZ2Decompressor()
+ else:
+ self.cmp = bz2.BZ2Compressor()
+
+ def __del__(self):
+ if hasattr(self, "closed") and not self.closed:
+ self.close()
+
+ def _init_write_gz(self):
+ """Initialize for writing with gzip compression.
+ """
+ self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED,
+ -self.zlib.MAX_WBITS,
+ self.zlib.DEF_MEM_LEVEL,
+ 0)
+ timestamp = struct.pack(" self.bufsize:
+ self.fileobj.write(self.buf[:self.bufsize])
+ self.buf = self.buf[self.bufsize:]
+
+ def close(self):
+ """Close the _Stream object. No operation should be
+ done on it afterwards.
+ """
+ if self.closed:
+ return
+
+ if self.mode == "w" and self.comptype != "tar":
+ self.buf += self.cmp.flush()
+
+ if self.mode == "w" and self.buf:
+ self.fileobj.write(self.buf)
+ self.buf = ""
+ if self.comptype == "gz":
+ # The native zlib crc is an unsigned 32-bit integer, but
+ # the Python wrapper implicitly casts that to a signed C
+ # long. So, on a 32-bit box self.crc may "look negative",
+ # while the same crc on a 64-bit box may "look positive".
+ # To avoid irksome warnings from the `struct` module, force
+ # it to look positive on all boxes.
+ self.fileobj.write(struct.pack("= 0:
+ blocks, remainder = divmod(pos - self.pos, self.bufsize)
+ for i in xrange(blocks):
+ self.read(self.bufsize)
+ self.read(remainder)
+ else:
+ raise StreamError("seeking backwards is not allowed")
+ return self.pos
+
+ def read(self, size=None):
+ """Return the next size number of bytes from the stream.
+ If size is not defined, return all bytes of the stream
+ up to EOF.
+ """
+ if size is None:
+ t = []
+ while True:
+ buf = self._read(self.bufsize)
+ if not buf:
+ break
+ t.append(buf)
+ buf = "".join(t)
+ else:
+ buf = self._read(size)
+ self.pos += len(buf)
+ return buf
+
+ def _read(self, size):
+ """Return size bytes from the stream.
+ """
+ if self.comptype == "tar":
+ return self.__read(size)
+
+ c = len(self.dbuf)
+ t = [self.dbuf]
+ while c < size:
+ buf = self.__read(self.bufsize)
+ if not buf:
+ break
+ try:
+ buf = self.cmp.decompress(buf)
+ except IOError:
+ raise ReadError("invalid compressed data")
+ t.append(buf)
+ c += len(buf)
+ t = "".join(t)
+ self.dbuf = t[size:]
+ return t[:size]
+
+ def __read(self, size):
+ """Return size bytes from stream. If internal buffer is empty,
+ read another block from the stream.
+ """
+ c = len(self.buf)
+ t = [self.buf]
+ while c < size:
+ buf = self.fileobj.read(self.bufsize)
+ if not buf:
+ break
+ t.append(buf)
+ c += len(buf)
+ t = "".join(t)
+ self.buf = t[size:]
+ return t[:size]
+# class _Stream
+
+class _StreamProxy(object):
+ """Small proxy class that enables transparent compression
+ detection for the Stream interface (mode 'r|*').
+ """
+
+ def __init__(self, fileobj):
+ self.fileobj = fileobj
+ self.buf = self.fileobj.read(BLOCKSIZE)
+
+ def read(self, size):
+ self.read = self.fileobj.read
+ return self.buf
+
+ def getcomptype(self):
+ if self.buf.startswith("\037\213\010"):
+ return "gz"
+ if self.buf.startswith("BZh91"):
+ return "bz2"
+ return "tar"
+
+ def close(self):
+ self.fileobj.close()
+# class StreamProxy
+
+class _BZ2Proxy(object):
+ """Small proxy class that enables external file object
+ support for "r:bz2" and "w:bz2" modes. This is actually
+ a workaround for a limitation in bz2 module's BZ2File
+ class which (unlike gzip.GzipFile) has no support for
+ a file object argument.
+ """
+
+ blocksize = 16 * 1024
+
+ def __init__(self, fileobj, mode):
+ self.fileobj = fileobj
+ self.mode = mode
+ self.name = getattr(self.fileobj, "name", None)
+ self.init()
+
+ def init(self):
+ import bz2
+ self.pos = 0
+ if self.mode == "r":
+ self.bz2obj = bz2.BZ2Decompressor()
+ self.fileobj.seek(0)
+ self.buf = ""
+ else:
+ self.bz2obj = bz2.BZ2Compressor()
+
+ def read(self, size):
+ b = [self.buf]
+ x = len(self.buf)
+ while x < size:
+ raw = self.fileobj.read(self.blocksize)
+ if not raw:
+ break
+ data = self.bz2obj.decompress(raw)
+ b.append(data)
+ x += len(data)
+ self.buf = "".join(b)
+
+ buf = self.buf[:size]
+ self.buf = self.buf[size:]
+ self.pos += len(buf)
+ return buf
+
+ def seek(self, pos):
+ if pos < self.pos:
+ self.init()
+ self.read(pos - self.pos)
+
+ def tell(self):
+ return self.pos
+
+ def write(self, data):
+ self.pos += len(data)
+ raw = self.bz2obj.compress(data)
+ self.fileobj.write(raw)
+
+ def close(self):
+ if self.mode == "w":
+ raw = self.bz2obj.flush()
+ self.fileobj.write(raw)
+# class _BZ2Proxy
+
+#------------------------
+# Extraction file object
+#------------------------
+class _FileInFile(object):
+ """A thin wrapper around an existing file object that
+ provides a part of its data as an individual file
+ object.
+ """
+
+ def __init__(self, fileobj, offset, size, sparse=None):
+ self.fileobj = fileobj
+ self.offset = offset
+ self.size = size
+ self.sparse = sparse
+ self.position = 0
+
+ def tell(self):
+ """Return the current file position.
+ """
+ return self.position
+
+ def seek(self, position):
+ """Seek to a position in the file.
+ """
+ self.position = position
+
+ def read(self, size=None):
+ """Read data from the file.
+ """
+ if size is None:
+ size = self.size - self.position
+ else:
+ size = min(size, self.size - self.position)
+
+ if self.sparse is None:
+ return self.readnormal(size)
+ else:
+ return self.readsparse(size)
+
+ def readnormal(self, size):
+ """Read operation for regular files.
+ """
+ self.fileobj.seek(self.offset + self.position)
+ self.position += size
+ return self.fileobj.read(size)
+
+ def readsparse(self, size):
+ """Read operation for sparse files.
+ """
+ data = []
+ while size > 0:
+ buf = self.readsparsesection(size)
+ if not buf:
+ break
+ size -= len(buf)
+ data.append(buf)
+ return "".join(data)
+
+ def readsparsesection(self, size):
+ """Read a single section of a sparse file.
+ """
+ section = self.sparse.find(self.position)
+
+ if section is None:
+ return ""
+
+ size = min(size, section.offset + section.size - self.position)
+
+ if isinstance(section, _data):
+ realpos = section.realpos + self.position - section.offset
+ self.fileobj.seek(self.offset + realpos)
+ self.position += size
+ return self.fileobj.read(size)
+ else:
+ self.position += size
+ return NUL * size
+#class _FileInFile
+
+
+class ExFileObject(object):
+ """File-like object for reading an archive member.
+ Is returned by TarFile.extractfile().
+ """
+ blocksize = 1024
+
+ def __init__(self, tarfile, tarinfo):
+ self.fileobj = _FileInFile(tarfile.fileobj,
+ tarinfo.offset_data,
+ tarinfo.size,
+ getattr(tarinfo, "sparse", None))
+ self.name = tarinfo.name
+ self.mode = "r"
+ self.closed = False
+ self.size = tarinfo.size
+
+ self.position = 0
+ self.buffer = ""
+
+ def read(self, size=None):
+ """Read at most size bytes from the file. If size is not
+ present or None, read all data until EOF is reached.
+ """
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+
+ buf = ""
+ if self.buffer:
+ if size is None:
+ buf = self.buffer
+ self.buffer = ""
+ else:
+ buf = self.buffer[:size]
+ self.buffer = self.buffer[size:]
+
+ if size is None:
+ buf += self.fileobj.read()
+ else:
+ buf += self.fileobj.read(size - len(buf))
+
+ self.position += len(buf)
+ return buf
+
+ def readline(self, size=-1):
+ """Read one entire line from the file. If size is present
+ and non-negative, return a string with at most that
+ size, which may be an incomplete line.
+ """
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+
+ if "\n" in self.buffer:
+ pos = self.buffer.find("\n") + 1
+ else:
+ buffers = [self.buffer]
+ while True:
+ buf = self.fileobj.read(self.blocksize)
+ buffers.append(buf)
+ if not buf or "\n" in buf:
+ self.buffer = "".join(buffers)
+ pos = self.buffer.find("\n") + 1
+ if pos == 0:
+ # no newline found.
+ pos = len(self.buffer)
+ break
+
+ if size != -1:
+ pos = min(size, pos)
+
+ buf = self.buffer[:pos]
+ self.buffer = self.buffer[pos:]
+ self.position += len(buf)
+ return buf
+
+ def readlines(self):
+ """Return a list with all remaining lines.
+ """
+ result = []
+ while True:
+ line = self.readline()
+ if not line: break
+ result.append(line)
+ return result
+
+ def tell(self):
+ """Return the current file position.
+ """
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+
+ return self.position
+
+ def seek(self, pos, whence=os.SEEK_SET):
+ """Seek to a position in the file.
+ """
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+
+ if whence == os.SEEK_SET:
+ self.position = min(max(pos, 0), self.size)
+ elif whence == os.SEEK_CUR:
+ if pos < 0:
+ self.position = max(self.position + pos, 0)
+ else:
+ self.position = min(self.position + pos, self.size)
+ elif whence == os.SEEK_END:
+ self.position = max(min(self.size + pos, self.size), 0)
+ else:
+ raise ValueError("Invalid argument")
+
+ self.buffer = ""
+ self.fileobj.seek(self.position)
+
+ def close(self):
+ """Close the file object.
+ """
+ self.closed = True
+
+ def __iter__(self):
+ """Get an iterator over the file's lines.
+ """
+ while True:
+ line = self.readline()
+ if not line:
+ break
+ yield line
+#class ExFileObject
+
+#------------------
+# Exported Classes
+#------------------
+class TarInfo(object):
+ """Informational class which holds the details about an
+ archive member given by a tar header block.
+ TarInfo objects are returned by TarFile.getmember(),
+ TarFile.getmembers() and TarFile.gettarinfo() and are
+ usually created internally.
+ """
+
+ def __init__(self, name=""):
+ """Construct a TarInfo object. name is the optional name
+ of the member.
+ """
+ self.name = name # member name
+ self.mode = 0644 # file permissions
+ self.uid = 0 # user id
+ self.gid = 0 # group id
+ self.size = 0 # file size
+ self.mtime = 0 # modification time
+ self.chksum = 0 # header checksum
+ self.type = REGTYPE # member type
+ self.linkname = "" # link name
+ self.uname = "" # user name
+ self.gname = "" # group name
+ self.devmajor = 0 # device major number
+ self.devminor = 0 # device minor number
+
+ self.offset = 0 # the tar header starts here
+ self.offset_data = 0 # the file's data starts here
+
+ self.pax_headers = {} # pax header information
+
+ # In pax headers the "name" and "linkname" field are called
+ # "path" and "linkpath".
+ def _getpath(self):
+ return self.name
+ def _setpath(self, name):
+ self.name = name
+ path = property(_getpath, _setpath)
+
+ def _getlinkpath(self):
+ return self.linkname
+ def _setlinkpath(self, linkname):
+ self.linkname = linkname
+ linkpath = property(_getlinkpath, _setlinkpath)
+
+ def __repr__(self):
+ return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self))
+
+ def get_info(self, encoding, errors):
+ """Return the TarInfo's attributes as a dictionary.
+ """
+ info = {
+ "name": self.name,
+ "mode": self.mode & 07777,
+ "uid": self.uid,
+ "gid": self.gid,
+ "size": self.size,
+ "mtime": self.mtime,
+ "chksum": self.chksum,
+ "type": self.type,
+ "linkname": self.linkname,
+ "uname": self.uname,
+ "gname": self.gname,
+ "devmajor": self.devmajor,
+ "devminor": self.devminor
+ }
+
+ if info["type"] == DIRTYPE and not info["name"].endswith("/"):
+ info["name"] += "/"
+
+ for key in ("name", "linkname", "uname", "gname"):
+ if type(info[key]) is unicode:
+ info[key] = info[key].encode(encoding, errors)
+
+ return info
+
+ def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="strict"):
+ """Return a tar header as a string of 512 byte blocks.
+ """
+ info = self.get_info(encoding, errors)
+
+ if format == USTAR_FORMAT:
+ return self.create_ustar_header(info)
+ elif format == GNU_FORMAT:
+ return self.create_gnu_header(info)
+ elif format == PAX_FORMAT:
+ return self.create_pax_header(info, encoding, errors)
+ else:
+ raise ValueError("invalid format")
+
+ def create_ustar_header(self, info):
+ """Return the object as a ustar header block.
+ """
+ info["magic"] = POSIX_MAGIC
+
+ if len(info["linkname"]) > LENGTH_LINK:
+ raise ValueError("linkname is too long")
+
+ if len(info["name"]) > LENGTH_NAME:
+ info["prefix"], info["name"] = self._posix_split_name(info["name"])
+
+ return self._create_header(info, USTAR_FORMAT)
+
+ def create_gnu_header(self, info):
+ """Return the object as a GNU header block sequence.
+ """
+ info["magic"] = GNU_MAGIC
+
+ buf = ""
+ if len(info["linkname"]) > LENGTH_LINK:
+ buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK)
+
+ if len(info["name"]) > LENGTH_NAME:
+ buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME)
+
+ return buf + self._create_header(info, GNU_FORMAT)
+
+ def create_pax_header(self, info, encoding, errors):
+ """Return the object as a ustar header block. If it cannot be
+ represented this way, prepend a pax extended header sequence
+ with supplement information.
+ """
+ info["magic"] = POSIX_MAGIC
+ pax_headers = self.pax_headers.copy()
+
+ # Test string fields for values that exceed the field length or cannot
+ # be represented in ASCII encoding.
+ for name, hname, length in (
+ ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK),
+ ("uname", "uname", 32), ("gname", "gname", 32)):
+
+ if hname in pax_headers:
+ # The pax header has priority.
+ continue
+
+ val = info[name].decode(encoding, errors)
+
+ # Try to encode the string as ASCII.
+ try:
+ val.encode("ascii")
+ except UnicodeEncodeError:
+ pax_headers[hname] = val
+ continue
+
+ if len(info[name]) > length:
+ pax_headers[hname] = val
+
+ # Test number fields for values that exceed the field limit or values
+ # that like to be stored as float.
+ for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)):
+ if name in pax_headers:
+ # The pax header has priority. Avoid overflow.
+ info[name] = 0
+ continue
+
+ val = info[name]
+ if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float):
+ pax_headers[name] = unicode(val)
+ info[name] = 0
+
+ # Create a pax extended header if necessary.
+ if pax_headers:
+ buf = self._create_pax_generic_header(pax_headers)
+ else:
+ buf = ""
+
+ return buf + self._create_header(info, USTAR_FORMAT)
+
+ @classmethod
+ def create_pax_global_header(cls, pax_headers):
+ """Return the object as a pax global header block sequence.
+ """
+ return cls._create_pax_generic_header(pax_headers, type=XGLTYPE)
+
+ def _posix_split_name(self, name):
+ """Split a name longer than 100 chars into a prefix
+ and a name part.
+ """
+ prefix = name[:LENGTH_PREFIX + 1]
+ while prefix and prefix[-1] != "/":
+ prefix = prefix[:-1]
+
+ name = name[len(prefix):]
+ prefix = prefix[:-1]
+
+ if not prefix or len(name) > LENGTH_NAME:
+ raise ValueError("name is too long")
+ return prefix, name
+
+ @staticmethod
+ def _create_header(info, format):
+ """Return a header block. info is a dictionary with file
+ information, format must be one of the *_FORMAT constants.
+ """
+ parts = [
+ stn(info.get("name", ""), 100),
+ itn(info.get("mode", 0) & 07777, 8, format),
+ itn(info.get("uid", 0), 8, format),
+ itn(info.get("gid", 0), 8, format),
+ itn(info.get("size", 0), 12, format),
+ itn(info.get("mtime", 0), 12, format),
+ " ", # checksum field
+ info.get("type", REGTYPE),
+ stn(info.get("linkname", ""), 100),
+ stn(info.get("magic", POSIX_MAGIC), 8),
+ stn(info.get("uname", ""), 32),
+ stn(info.get("gname", ""), 32),
+ itn(info.get("devmajor", 0), 8, format),
+ itn(info.get("devminor", 0), 8, format),
+ stn(info.get("prefix", ""), 155)
+ ]
+
+ buf = struct.pack("%ds" % BLOCKSIZE, "".join(parts))
+ chksum = calc_chksums(buf[-BLOCKSIZE:])[0]
+ buf = buf[:-364] + "%06o\0" % chksum + buf[-357:]
+ return buf
+
+ @staticmethod
+ def _create_payload(payload):
+ """Return the string payload filled with zero bytes
+ up to the next 512 byte border.
+ """
+ blocks, remainder = divmod(len(payload), BLOCKSIZE)
+ if remainder > 0:
+ payload += (BLOCKSIZE - remainder) * NUL
+ return payload
+
+ @classmethod
+ def _create_gnu_long_header(cls, name, type):
+ """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence
+ for name.
+ """
+ name += NUL
+
+ info = {}
+ info["name"] = "././@LongLink"
+ info["type"] = type
+ info["size"] = len(name)
+ info["magic"] = GNU_MAGIC
+
+ # create extended header + name blocks.
+ return cls._create_header(info, USTAR_FORMAT) + \
+ cls._create_payload(name)
+
+ @classmethod
+ def _create_pax_generic_header(cls, pax_headers, type=XHDTYPE):
+ """Return a POSIX.1-2001 extended or global header sequence
+ that contains a list of keyword, value pairs. The values
+ must be unicode objects.
+ """
+ records = []
+ for keyword, value in pax_headers.iteritems():
+ keyword = keyword.encode("utf8")
+ value = value.encode("utf8")
+ l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n'
+ n = p = 0
+ while True:
+ n = l + len(str(p))
+ if n == p:
+ break
+ p = n
+ records.append("%d %s=%s\n" % (p, keyword, value))
+ records = "".join(records)
+
+ # We use a hardcoded "././@PaxHeader" name like star does
+ # instead of the one that POSIX recommends.
+ info = {}
+ info["name"] = "././@PaxHeader"
+ info["type"] = type
+ info["size"] = len(records)
+ info["magic"] = POSIX_MAGIC
+
+ # Create pax header + record blocks.
+ return cls._create_header(info, USTAR_FORMAT) + \
+ cls._create_payload(records)
+
+ @classmethod
+ def frombuf(cls, buf):
+ """Construct a TarInfo object from a 512 byte string buffer.
+ """
+ if len(buf) == 0:
+ raise EmptyHeaderError("empty header")
+ if len(buf) != BLOCKSIZE:
+ raise TruncatedHeaderError("truncated header")
+ if buf.count(NUL) == BLOCKSIZE:
+ raise EOFHeaderError("end of file header")
+
+ chksum = nti(buf[148:156])
+ if chksum not in calc_chksums(buf):
+ raise InvalidHeaderError("bad checksum")
+
+ obj = cls()
+ obj.buf = buf
+ obj.name = nts(buf[0:100])
+ obj.mode = nti(buf[100:108])
+ obj.uid = nti(buf[108:116])
+ obj.gid = nti(buf[116:124])
+ obj.size = nti(buf[124:136])
+ obj.mtime = nti(buf[136:148])
+ obj.chksum = chksum
+ obj.type = buf[156:157]
+ obj.linkname = nts(buf[157:257])
+ obj.uname = nts(buf[265:297])
+ obj.gname = nts(buf[297:329])
+ obj.devmajor = nti(buf[329:337])
+ obj.devminor = nti(buf[337:345])
+ prefix = nts(buf[345:500])
+
+ # Old V7 tar format represents a directory as a regular
+ # file with a trailing slash.
+ if obj.type == AREGTYPE and obj.name.endswith("/"):
+ obj.type = DIRTYPE
+
+ # Remove redundant slashes from directories.
+ if obj.isdir():
+ obj.name = obj.name.rstrip("/")
+
+ # Reconstruct a ustar longname.
+ if prefix and obj.type not in GNU_TYPES:
+ obj.name = prefix + "/" + obj.name
+ return obj
+
+ @classmethod
+ def fromtarfile(cls, tarfile):
+ """Return the next TarInfo object from TarFile object
+ tarfile.
+ """
+ buf = tarfile.fileobj.read(BLOCKSIZE)
+ obj = cls.frombuf(buf)
+ obj.offset = tarfile.fileobj.tell() - BLOCKSIZE
+ return obj._proc_member(tarfile)
+
+ #--------------------------------------------------------------------------
+ # The following are methods that are called depending on the type of a
+ # member. The entry point is _proc_member() which can be overridden in a
+ # subclass to add custom _proc_*() methods. A _proc_*() method MUST
+ # implement the following
+ # operations:
+ # 1. Set self.offset_data to the position where the data blocks begin,
+ # if there is data that follows.
+ # 2. Set tarfile.offset to the position where the next member's header will
+ # begin.
+ # 3. Return self or another valid TarInfo object.
+ def _proc_member(self, tarfile):
+ """Choose the right processing method depending on
+ the type and call it.
+ """
+ if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK):
+ return self._proc_gnulong(tarfile)
+ elif self.type == GNUTYPE_SPARSE:
+ return self._proc_sparse(tarfile)
+ elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE):
+ return self._proc_pax(tarfile)
+ else:
+ return self._proc_builtin(tarfile)
+
+ def _proc_builtin(self, tarfile):
+ """Process a builtin type or an unknown type which
+ will be treated as a regular file.
+ """
+ self.offset_data = tarfile.fileobj.tell()
+ offset = self.offset_data
+ if self.isreg() or self.type not in SUPPORTED_TYPES:
+ # Skip the following data blocks.
+ offset += self._block(self.size)
+ tarfile.offset = offset
+
+ # Patch the TarInfo object with saved global
+ # header information.
+ self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors)
+
+ return self
+
+ def _proc_gnulong(self, tarfile):
+ """Process the blocks that hold a GNU longname
+ or longlink member.
+ """
+ buf = tarfile.fileobj.read(self._block(self.size))
+
+ # Fetch the next header and process it.
+ try:
+ next = self.fromtarfile(tarfile)
+ except HeaderError:
+ raise SubsequentHeaderError("missing or bad subsequent header")
+
+ # Patch the TarInfo object from the next header with
+ # the longname information.
+ next.offset = self.offset
+ if self.type == GNUTYPE_LONGNAME:
+ next.name = nts(buf)
+ elif self.type == GNUTYPE_LONGLINK:
+ next.linkname = nts(buf)
+
+ return next
+
+ def _proc_sparse(self, tarfile):
+ """Process a GNU sparse header plus extra headers.
+ """
+ buf = self.buf
+ sp = _ringbuffer()
+ pos = 386
+ lastpos = 0L
+ realpos = 0L
+ # There are 4 possible sparse structs in the
+ # first header.
+ for i in xrange(4):
+ try:
+ offset = nti(buf[pos:pos + 12])
+ numbytes = nti(buf[pos + 12:pos + 24])
+ except ValueError:
+ break
+ if offset > lastpos:
+ sp.append(_hole(lastpos, offset - lastpos))
+ sp.append(_data(offset, numbytes, realpos))
+ realpos += numbytes
+ lastpos = offset + numbytes
+ pos += 24
+
+ isextended = ord(buf[482])
+ origsize = nti(buf[483:495])
+
+ # If the isextended flag is given,
+ # there are extra headers to process.
+ while isextended == 1:
+ buf = tarfile.fileobj.read(BLOCKSIZE)
+ pos = 0
+ for i in xrange(21):
+ try:
+ offset = nti(buf[pos:pos + 12])
+ numbytes = nti(buf[pos + 12:pos + 24])
+ except ValueError:
+ break
+ if offset > lastpos:
+ sp.append(_hole(lastpos, offset - lastpos))
+ sp.append(_data(offset, numbytes, realpos))
+ realpos += numbytes
+ lastpos = offset + numbytes
+ pos += 24
+ isextended = ord(buf[504])
+
+ if lastpos < origsize:
+ sp.append(_hole(lastpos, origsize - lastpos))
+
+ self.sparse = sp
+
+ self.offset_data = tarfile.fileobj.tell()
+ tarfile.offset = self.offset_data + self._block(self.size)
+ self.size = origsize
+
+ return self
+
+ def _proc_pax(self, tarfile):
+ """Process an extended or global header as described in
+ POSIX.1-2001.
+ """
+ # Read the header information.
+ buf = tarfile.fileobj.read(self._block(self.size))
+
+ # A pax header stores supplemental information for either
+ # the following file (extended) or all following files
+ # (global).
+ if self.type == XGLTYPE:
+ pax_headers = tarfile.pax_headers
+ else:
+ pax_headers = tarfile.pax_headers.copy()
+
+ # Parse pax header information. A record looks like that:
+ # "%d %s=%s\n" % (length, keyword, value). length is the size
+ # of the complete record including the length field itself and
+ # the newline. keyword and value are both UTF-8 encoded strings.
+ regex = re.compile(r"(\d+) ([^=]+)=", re.U)
+ pos = 0
+ while True:
+ match = regex.match(buf, pos)
+ if not match:
+ break
+
+ length, keyword = match.groups()
+ length = int(length)
+ value = buf[match.end(2) + 1:match.start(1) + length - 1]
+
+ keyword = keyword.decode("utf8")
+ value = value.decode("utf8")
+
+ pax_headers[keyword] = value
+ pos += length
+
+ # Fetch the next header.
+ try:
+ next = self.fromtarfile(tarfile)
+ except HeaderError:
+ raise SubsequentHeaderError("missing or bad subsequent header")
+
+ if self.type in (XHDTYPE, SOLARIS_XHDTYPE):
+ # Patch the TarInfo object with the extended header info.
+ next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors)
+ next.offset = self.offset
+
+ if "size" in pax_headers:
+ # If the extended header replaces the size field,
+ # we need to recalculate the offset where the next
+ # header starts.
+ offset = next.offset_data
+ if next.isreg() or next.type not in SUPPORTED_TYPES:
+ offset += next._block(next.size)
+ tarfile.offset = offset
+
+ return next
+
+ def _apply_pax_info(self, pax_headers, encoding, errors):
+ """Replace fields with supplemental information from a previous
+ pax extended or global header.
+ """
+ for keyword, value in pax_headers.iteritems():
+ if keyword not in PAX_FIELDS:
+ continue
+
+ if keyword == "path":
+ value = value.rstrip("/")
+
+ if keyword in PAX_NUMBER_FIELDS:
+ try:
+ value = PAX_NUMBER_FIELDS[keyword](value)
+ except ValueError:
+ value = 0
+ else:
+ value = uts(value, encoding, errors)
+
+ setattr(self, keyword, value)
+
+ self.pax_headers = pax_headers.copy()
+
+ def _block(self, count):
+ """Round up a byte count by BLOCKSIZE and return it,
+ e.g. _block(834) => 1024.
+ """
+ blocks, remainder = divmod(count, BLOCKSIZE)
+ if remainder:
+ blocks += 1
+ return blocks * BLOCKSIZE
+
+ def isreg(self):
+ return self.type in REGULAR_TYPES
+ def isfile(self):
+ return self.isreg()
+ def isdir(self):
+ return self.type == DIRTYPE
+ def issym(self):
+ return self.type == SYMTYPE
+ def islnk(self):
+ return self.type == LNKTYPE
+ def ischr(self):
+ return self.type == CHRTYPE
+ def isblk(self):
+ return self.type == BLKTYPE
+ def isfifo(self):
+ return self.type == FIFOTYPE
+ def issparse(self):
+ return self.type == GNUTYPE_SPARSE
+ def isdev(self):
+ return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE)
+# class TarInfo
+
+class TarFile(object):
+ """The TarFile Class provides an interface to tar archives.
+ """
+
+ debug = 0 # May be set from 0 (no msgs) to 3 (all msgs)
+
+ dereference = False # If true, add content of linked file to the
+ # tar file, else the link.
+
+ ignore_zeros = False # If true, skips empty or invalid blocks and
+ # continues processing.
+
+ errorlevel = 1 # If 0, fatal errors only appear in debug
+ # messages (if debug >= 0). If > 0, errors
+ # are passed to the caller as exceptions.
+
+ format = DEFAULT_FORMAT # The format to use when creating an archive.
+
+ encoding = ENCODING # Encoding for 8-bit character strings.
+
+ errors = None # Error handler for unicode conversion.
+
+ tarinfo = TarInfo # The default TarInfo class to use.
+
+ fileobject = ExFileObject # The default ExFileObject class to use.
+
+ def __init__(self, name=None, mode="r", fileobj=None, format=None,
+ tarinfo=None, dereference=None, ignore_zeros=None, encoding=None,
+ errors=None, pax_headers=None, debug=None, errorlevel=None):
+ """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to
+ read from an existing archive, 'a' to append data to an existing
+ file or 'w' to create a new file overwriting an existing one. `mode'
+ defaults to 'r'.
+ If `fileobj' is given, it is used for reading or writing data. If it
+ can be determined, `mode' is overridden by `fileobj's mode.
+ `fileobj' is not closed, when TarFile is closed.
+ """
+ if len(mode) > 1 or mode not in "raw":
+ raise ValueError("mode must be 'r', 'a' or 'w'")
+ self.mode = mode
+ self._mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode]
+
+ if not fileobj:
+ if self.mode == "a" and not os.path.exists(name):
+ # Create nonexistent files in append mode.
+ self.mode = "w"
+ self._mode = "wb"
+ fileobj = bltn_open(name, self._mode)
+ self._extfileobj = False
+ else:
+ if name is None and hasattr(fileobj, "name"):
+ name = fileobj.name
+ if hasattr(fileobj, "mode"):
+ self._mode = fileobj.mode
+ self._extfileobj = True
+ self.name = os.path.abspath(name) if name else None
+ self.fileobj = fileobj
+
+ # Init attributes.
+ if format is not None:
+ self.format = format
+ if tarinfo is not None:
+ self.tarinfo = tarinfo
+ if dereference is not None:
+ self.dereference = dereference
+ if ignore_zeros is not None:
+ self.ignore_zeros = ignore_zeros
+ if encoding is not None:
+ self.encoding = encoding
+
+ if errors is not None:
+ self.errors = errors
+ elif mode == "r":
+ self.errors = "utf-8"
+ else:
+ self.errors = "strict"
+
+ if pax_headers is not None and self.format == PAX_FORMAT:
+ self.pax_headers = pax_headers
+ else:
+ self.pax_headers = {}
+
+ if debug is not None:
+ self.debug = debug
+ if errorlevel is not None:
+ self.errorlevel = errorlevel
+
+ # Init datastructures.
+ self.closed = False
+ self.members = [] # list of members as TarInfo objects
+ self._loaded = False # flag if all members have been read
+ self.offset = self.fileobj.tell()
+ # current position in the archive file
+ self.inodes = {} # dictionary caching the inodes of
+ # archive members already added
+
+ try:
+ if self.mode == "r":
+ self.firstmember = None
+ self.firstmember = self.next()
+
+ if self.mode == "a":
+ # Move to the end of the archive,
+ # before the first empty block.
+ while True:
+ self.fileobj.seek(self.offset)
+ try:
+ tarinfo = self.tarinfo.fromtarfile(self)
+ self.members.append(tarinfo)
+ except EOFHeaderError:
+ self.fileobj.seek(self.offset)
+ break
+ except HeaderError, e:
+ raise ReadError(str(e))
+
+ if self.mode in "aw":
+ self._loaded = True
+
+ if self.pax_headers:
+ buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy())
+ self.fileobj.write(buf)
+ self.offset += len(buf)
+ except:
+ if not self._extfileobj:
+ self.fileobj.close()
+ self.closed = True
+ raise
+
+ def _getposix(self):
+ return self.format == USTAR_FORMAT
+ def _setposix(self, value):
+ import warnings
+ warnings.warn("use the format attribute instead", DeprecationWarning,
+ 2)
+ if value:
+ self.format = USTAR_FORMAT
+ else:
+ self.format = GNU_FORMAT
+ posix = property(_getposix, _setposix)
+
+ #--------------------------------------------------------------------------
+ # Below are the classmethods which act as alternate constructors to the
+ # TarFile class. The open() method is the only one that is needed for
+ # public use; it is the "super"-constructor and is able to select an
+ # adequate "sub"-constructor for a particular compression using the mapping
+ # from OPEN_METH.
+ #
+ # This concept allows one to subclass TarFile without losing the comfort of
+ # the super-constructor. A sub-constructor is registered and made available
+ # by adding it to the mapping in OPEN_METH.
+
+ @classmethod
+ def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs):
+ """Open a tar archive for reading, writing or appending. Return
+ an appropriate TarFile class.
+
+ mode:
+ 'r' or 'r:*' open for reading with transparent compression
+ 'r:' open for reading exclusively uncompressed
+ 'r:gz' open for reading with gzip compression
+ 'r:bz2' open for reading with bzip2 compression
+ 'a' or 'a:' open for appending, creating the file if necessary
+ 'w' or 'w:' open for writing without compression
+ 'w:gz' open for writing with gzip compression
+ 'w:bz2' open for writing with bzip2 compression
+
+ 'r|*' open a stream of tar blocks with transparent compression
+ 'r|' open an uncompressed stream of tar blocks for reading
+ 'r|gz' open a gzip compressed stream of tar blocks
+ 'r|bz2' open a bzip2 compressed stream of tar blocks
+ 'w|' open an uncompressed stream for writing
+ 'w|gz' open a gzip compressed stream for writing
+ 'w|bz2' open a bzip2 compressed stream for writing
+ """
+
+ if not name and not fileobj:
+ raise ValueError("nothing to open")
+
+ if mode in ("r", "r:*"):
+ # Find out which *open() is appropriate for opening the file.
+ for comptype in cls.OPEN_METH:
+ func = getattr(cls, cls.OPEN_METH[comptype])
+ if fileobj is not None:
+ saved_pos = fileobj.tell()
+ try:
+ return func(name, "r", fileobj, **kwargs)
+ except (ReadError, CompressionError), e:
+ if fileobj is not None:
+ fileobj.seek(saved_pos)
+ continue
+ raise ReadError("file could not be opened successfully")
+
+ elif ":" in mode:
+ filemode, comptype = mode.split(":", 1)
+ filemode = filemode or "r"
+ comptype = comptype or "tar"
+
+ # Select the *open() function according to
+ # given compression.
+ if comptype in cls.OPEN_METH:
+ func = getattr(cls, cls.OPEN_METH[comptype])
+ else:
+ raise CompressionError("unknown compression type %r" % comptype)
+ return func(name, filemode, fileobj, **kwargs)
+
+ elif "|" in mode:
+ filemode, comptype = mode.split("|", 1)
+ filemode = filemode or "r"
+ comptype = comptype or "tar"
+
+ if filemode not in "rw":
+ raise ValueError("mode must be 'r' or 'w'")
+
+ t = cls(name, filemode,
+ _Stream(name, filemode, comptype, fileobj, bufsize),
+ **kwargs)
+ t._extfileobj = False
+ return t
+
+ elif mode in "aw":
+ return cls.taropen(name, mode, fileobj, **kwargs)
+
+ raise ValueError("undiscernible mode")
+
+ @classmethod
+ def taropen(cls, name, mode="r", fileobj=None, **kwargs):
+ """Open uncompressed tar archive name for reading or writing.
+ """
+ if len(mode) > 1 or mode not in "raw":
+ raise ValueError("mode must be 'r', 'a' or 'w'")
+ return cls(name, mode, fileobj, **kwargs)
+
+ @classmethod
+ def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
+ """Open gzip compressed tar archive name for reading or writing.
+ Appending is not allowed.
+ """
+ if len(mode) > 1 or mode not in "rw":
+ raise ValueError("mode must be 'r' or 'w'")
+
+ try:
+ import gzip
+ gzip.GzipFile
+ except (ImportError, AttributeError):
+ raise CompressionError("gzip module is not available")
+
+ if fileobj is None:
+ fileobj = bltn_open(name, mode + "b")
+
+ try:
+ t = cls.taropen(name, mode,
+ gzip.GzipFile(name, mode, compresslevel, fileobj),
+ **kwargs)
+ except IOError:
+ raise ReadError("not a gzip file")
+ t._extfileobj = False
+ return t
+
+ @classmethod
+ def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
+ """Open bzip2 compressed tar archive name for reading or writing.
+ Appending is not allowed.
+ """
+ if len(mode) > 1 or mode not in "rw":
+ raise ValueError("mode must be 'r' or 'w'.")
+
+ try:
+ import bz2
+ except ImportError:
+ raise CompressionError("bz2 module is not available")
+
+ if fileobj is not None:
+ fileobj = _BZ2Proxy(fileobj, mode)
+ else:
+ fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
+
+ try:
+ t = cls.taropen(name, mode, fileobj, **kwargs)
+ except (IOError, EOFError):
+ raise ReadError("not a bzip2 file")
+ t._extfileobj = False
+ return t
+
+ # All *open() methods are registered here.
+ OPEN_METH = {
+ "tar": "taropen", # uncompressed tar
+ "gz": "gzopen", # gzip compressed tar
+ "bz2": "bz2open" # bzip2 compressed tar
+ }
+
+ #--------------------------------------------------------------------------
+ # The public methods which TarFile provides:
+
+ def close(self):
+ """Close the TarFile. In write-mode, two finishing zero blocks are
+ appended to the archive.
+ """
+ if self.closed:
+ return
+
+ if self.mode in "aw":
+ self.fileobj.write(NUL * (BLOCKSIZE * 2))
+ self.offset += (BLOCKSIZE * 2)
+ # fill up the end with zero-blocks
+ # (like option -b20 for tar does)
+ blocks, remainder = divmod(self.offset, RECORDSIZE)
+ if remainder > 0:
+ self.fileobj.write(NUL * (RECORDSIZE - remainder))
+
+ if not self._extfileobj:
+ self.fileobj.close()
+ self.closed = True
+
+ def getmember(self, name):
+ """Return a TarInfo object for member `name'. If `name' can not be
+ found in the archive, KeyError is raised. If a member occurs more
+ than once in the archive, its last occurrence is assumed to be the
+ most up-to-date version.
+ """
+ tarinfo = self._getmember(name)
+ if tarinfo is None:
+ raise KeyError("filename %r not found" % name)
+ return tarinfo
+
+ def getmembers(self):
+ """Return the members of the archive as a list of TarInfo objects. The
+ list has the same order as the members in the archive.
+ """
+ self._check()
+ if not self._loaded: # if we want to obtain a list of
+ self._load() # all members, we first have to
+ # scan the whole archive.
+ return self.members
+
+ def getnames(self):
+ """Return the members of the archive as a list of their names. It has
+ the same order as the list returned by getmembers().
+ """
+ return [tarinfo.name for tarinfo in self.getmembers()]
+
+ def gettarinfo(self, name=None, arcname=None, fileobj=None):
+ """Create a TarInfo object for either the file `name' or the file
+ object `fileobj' (using os.fstat on its file descriptor). You can
+ modify some of the TarInfo's attributes before you add it using
+ addfile(). If given, `arcname' specifies an alternative name for the
+ file in the archive.
+ """
+ self._check("aw")
+
+ # When fileobj is given, replace name by
+ # fileobj's real name.
+ if fileobj is not None:
+ name = fileobj.name
+
+ # Building the name of the member in the archive.
+ # Backward slashes are converted to forward slashes,
+ # Absolute paths are turned to relative paths.
+ if arcname is None:
+ arcname = name
+ drv, arcname = os.path.splitdrive(arcname)
+ arcname = arcname.replace(os.sep, "/")
+ arcname = arcname.lstrip("/")
+
+ # Now, fill the TarInfo object with
+ # information specific for the file.
+ tarinfo = self.tarinfo()
+ tarinfo.tarfile = self
+
+ # Use os.stat or os.lstat, depending on platform
+ # and if symlinks shall be resolved.
+ if fileobj is None:
+ if hasattr(os, "lstat") and not self.dereference:
+ statres = os.lstat(name)
+ else:
+ statres = os.stat(name)
+ else:
+ statres = os.fstat(fileobj.fileno())
+ linkname = ""
+
+ stmd = statres.st_mode
+ if stat.S_ISREG(stmd):
+ inode = (statres.st_ino, statres.st_dev)
+ if not self.dereference and statres.st_nlink > 1 and \
+ inode in self.inodes and arcname != self.inodes[inode]:
+ # Is it a hardlink to an already
+ # archived file?
+ type = LNKTYPE
+ linkname = self.inodes[inode]
+ else:
+ # The inode is added only if its valid.
+ # For win32 it is always 0.
+ type = REGTYPE
+ if inode[0]:
+ self.inodes[inode] = arcname
+ elif stat.S_ISDIR(stmd):
+ type = DIRTYPE
+ elif stat.S_ISFIFO(stmd):
+ type = FIFOTYPE
+ elif stat.S_ISLNK(stmd):
+ type = SYMTYPE
+ linkname = os.readlink(name)
+ elif stat.S_ISCHR(stmd):
+ type = CHRTYPE
+ elif stat.S_ISBLK(stmd):
+ type = BLKTYPE
+ else:
+ return None
+
+ # Fill the TarInfo object with all
+ # information we can get.
+ tarinfo.name = arcname
+ tarinfo.mode = stmd
+ tarinfo.uid = statres.st_uid
+ tarinfo.gid = statres.st_gid
+ if type == REGTYPE:
+ tarinfo.size = statres.st_size
+ else:
+ tarinfo.size = 0L
+ tarinfo.mtime = statres.st_mtime
+ tarinfo.type = type
+ tarinfo.linkname = linkname
+ if pwd:
+ try:
+ tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
+ except KeyError:
+ pass
+ if grp:
+ try:
+ tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
+ except KeyError:
+ pass
+
+ if type in (CHRTYPE, BLKTYPE):
+ if hasattr(os, "major") and hasattr(os, "minor"):
+ tarinfo.devmajor = os.major(statres.st_rdev)
+ tarinfo.devminor = os.minor(statres.st_rdev)
+ return tarinfo
+
+ def list(self, verbose=True):
+ """Print a table of contents to sys.stdout. If `verbose' is False, only
+ the names of the members are printed. If it is True, an `ls -l'-like
+ output is produced.
+ """
+ self._check()
+
+ for tarinfo in self:
+ if verbose:
+ print filemode(tarinfo.mode),
+ print "%s/%s" % (tarinfo.uname or tarinfo.uid,
+ tarinfo.gname or tarinfo.gid),
+ if tarinfo.ischr() or tarinfo.isblk():
+ print "%10s" % ("%d,%d" \
+ % (tarinfo.devmajor, tarinfo.devminor)),
+ else:
+ print "%10d" % tarinfo.size,
+ print "%d-%02d-%02d %02d:%02d:%02d" \
+ % time.localtime(tarinfo.mtime)[:6],
+
+ print tarinfo.name + ("/" if tarinfo.isdir() else ""),
+
+ if verbose:
+ if tarinfo.issym():
+ print "->", tarinfo.linkname,
+ if tarinfo.islnk():
+ print "link to", tarinfo.linkname,
+ print
+
+ def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
+ """Add the file `name' to the archive. `name' may be any type of file
+ (directory, fifo, symbolic link, etc.). If given, `arcname'
+ specifies an alternative name for the file in the archive.
+ Directories are added recursively by default. This can be avoided by
+ setting `recursive' to False. `exclude' is a function that should
+ return True for each filename to be excluded. `filter' is a function
+ that expects a TarInfo object argument and returns the changed
+ TarInfo object, if it returns None the TarInfo object will be
+ excluded from the archive.
+ """
+ self._check("aw")
+
+ if arcname is None:
+ arcname = name
+
+ # Exclude pathnames.
+ if exclude is not None:
+ import warnings
+ warnings.warn("use the filter argument instead",
+ DeprecationWarning, 2)
+ if exclude(name):
+ self._dbg(2, "tarfile: Excluded %r" % name)
+ return
+
+ # Skip if somebody tries to archive the archive...
+ if self.name is not None and os.path.abspath(name) == self.name:
+ self._dbg(2, "tarfile: Skipped %r" % name)
+ return
+
+ self._dbg(1, name)
+
+ # Create a TarInfo object from the file.
+ tarinfo = self.gettarinfo(name, arcname)
+
+ if tarinfo is None:
+ self._dbg(1, "tarfile: Unsupported type %r" % name)
+ return
+
+ # Change or exclude the TarInfo object.
+ if filter is not None:
+ tarinfo = filter(tarinfo)
+ if tarinfo is None:
+ self._dbg(2, "tarfile: Excluded %r" % name)
+ return
+
+ # Append the tar header and data to the archive.
+ if tarinfo.isreg():
+ f = bltn_open(name, "rb")
+ self.addfile(tarinfo, f)
+ f.close()
+
+ elif tarinfo.isdir():
+ self.addfile(tarinfo)
+ if recursive:
+ for f in os.listdir(name):
+ self.add(os.path.join(name, f), os.path.join(arcname, f),
+ recursive, exclude, filter)
+
+ else:
+ self.addfile(tarinfo)
+
+ def addfile(self, tarinfo, fileobj=None):
+ """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is
+ given, tarinfo.size bytes are read from it and added to the archive.
+ You can create TarInfo objects using gettarinfo().
+ On Windows platforms, `fileobj' should always be opened with mode
+ 'rb' to avoid irritation about the file size.
+ """
+ self._check("aw")
+
+ tarinfo = copy.copy(tarinfo)
+
+ buf = tarinfo.tobuf(self.format, self.encoding, self.errors)
+ self.fileobj.write(buf)
+ self.offset += len(buf)
+
+ # If there's data to follow, append it.
+ if fileobj is not None:
+ copyfileobj(fileobj, self.fileobj, tarinfo.size)
+ blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
+ if remainder > 0:
+ self.fileobj.write(NUL * (BLOCKSIZE - remainder))
+ blocks += 1
+ self.offset += blocks * BLOCKSIZE
+
+ self.members.append(tarinfo)
+
+ def extractall(self, path=".", members=None):
+ """Extract all members from the archive to the current working
+ directory and set owner, modification time and permissions on
+ directories afterwards. `path' specifies a different directory
+ to extract to. `members' is optional and must be a subset of the
+ list returned by getmembers().
+ """
+ directories = []
+
+ if members is None:
+ members = self
+
+ for tarinfo in members:
+ if tarinfo.isdir():
+ # Extract directories with a safe mode.
+ directories.append(tarinfo)
+ tarinfo = copy.copy(tarinfo)
+ tarinfo.mode = 0700
+ self.extract(tarinfo, path)
+
+ # Reverse sort directories.
+ directories.sort(key=operator.attrgetter('name'))
+ directories.reverse()
+
+ # Set correct owner, mtime and filemode on directories.
+ for tarinfo in directories:
+ dirpath = os.path.join(path, tarinfo.name)
+ try:
+ self.chown(tarinfo, dirpath)
+ self.utime(tarinfo, dirpath)
+ self.chmod(tarinfo, dirpath)
+ except ExtractError, e:
+ if self.errorlevel > 1:
+ raise
+ else:
+ self._dbg(1, "tarfile: %s" % e)
+
+ def extract(self, member, path=""):
+ """Extract a member from the archive to the current working directory,
+ using its full name. Its file information is extracted as accurately
+ as possible. `member' may be a filename or a TarInfo object. You can
+ specify a different directory using `path'.
+ """
+ self._check("r")
+
+ if isinstance(member, basestring):
+ tarinfo = self.getmember(member)
+ else:
+ tarinfo = member
+
+ # Prepare the link target for makelink().
+ if tarinfo.islnk():
+ tarinfo._link_target = os.path.join(path, tarinfo.linkname)
+
+ try:
+ self._extract_member(tarinfo, os.path.join(path, tarinfo.name))
+ except EnvironmentError, e:
+ if self.errorlevel > 0:
+ raise
+ else:
+ if e.filename is None:
+ self._dbg(1, "tarfile: %s" % e.strerror)
+ else:
+ self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
+ except ExtractError, e:
+ if self.errorlevel > 1:
+ raise
+ else:
+ self._dbg(1, "tarfile: %s" % e)
+
+ def extractfile(self, member):
+ """Extract a member from the archive as a file object. `member' may be
+ a filename or a TarInfo object. If `member' is a regular file, a
+ file-like object is returned. If `member' is a link, a file-like
+ object is constructed from the link's target. If `member' is none of
+ the above, None is returned.
+ The file-like object is read-only and provides the following
+ methods: read(), readline(), readlines(), seek() and tell()
+ """
+ self._check("r")
+
+ if isinstance(member, basestring):
+ tarinfo = self.getmember(member)
+ else:
+ tarinfo = member
+
+ if tarinfo.isreg():
+ return self.fileobject(self, tarinfo)
+
+ elif tarinfo.type not in SUPPORTED_TYPES:
+ # If a member's type is unknown, it is treated as a
+ # regular file.
+ return self.fileobject(self, tarinfo)
+
+ elif tarinfo.islnk() or tarinfo.issym():
+ if isinstance(self.fileobj, _Stream):
+ # A small but ugly workaround for the case that someone tries
+ # to extract a (sym)link as a file-object from a non-seekable
+ # stream of tar blocks.
+ raise StreamError("cannot extract (sym)link as file object")
+ else:
+ # A (sym)link's file object is its target's file object.
+ return self.extractfile(self._find_link_target(tarinfo))
+ else:
+ # If there's no data associated with the member (directory, chrdev,
+ # blkdev, etc.), return None instead of a file object.
+ return None
+
+ def _extract_member(self, tarinfo, targetpath):
+ """Extract the TarInfo object tarinfo to a physical
+ file called targetpath.
+ """
+ # Fetch the TarInfo object for the given name
+ # and build the destination pathname, replacing
+ # forward slashes to platform specific separators.
+ targetpath = targetpath.rstrip("/")
+ targetpath = targetpath.replace("/", os.sep)
+
+ # Create all upper directories.
+ upperdirs = os.path.dirname(targetpath)
+ if upperdirs and not os.path.exists(upperdirs):
+ # Create directories that are not part of the archive with
+ # default permissions.
+ os.makedirs(upperdirs)
+
+ if tarinfo.islnk() or tarinfo.issym():
+ self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname))
+ else:
+ self._dbg(1, tarinfo.name)
+
+ if tarinfo.isreg():
+ self.makefile(tarinfo, targetpath)
+ elif tarinfo.isdir():
+ self.makedir(tarinfo, targetpath)
+ elif tarinfo.isfifo():
+ self.makefifo(tarinfo, targetpath)
+ elif tarinfo.ischr() or tarinfo.isblk():
+ self.makedev(tarinfo, targetpath)
+ elif tarinfo.islnk() or tarinfo.issym():
+ self.makelink(tarinfo, targetpath)
+ elif tarinfo.type not in SUPPORTED_TYPES:
+ self.makeunknown(tarinfo, targetpath)
+ else:
+ self.makefile(tarinfo, targetpath)
+
+ self.chown(tarinfo, targetpath)
+ if not tarinfo.issym():
+ self.chmod(tarinfo, targetpath)
+ self.utime(tarinfo, targetpath)
+
+ #--------------------------------------------------------------------------
+ # Below are the different file methods. They are called via
+ # _extract_member() when extract() is called. They can be replaced in a
+ # subclass to implement other functionality.
+
+ def makedir(self, tarinfo, targetpath):
+ """Make a directory called targetpath.
+ """
+ try:
+ # Use a safe mode for the directory, the real mode is set
+ # later in _extract_member().
+ os.mkdir(targetpath, 0700)
+ except EnvironmentError, e:
+ if e.errno != errno.EEXIST:
+ raise
+
+ def makefile(self, tarinfo, targetpath):
+ """Make a file called targetpath.
+ """
+ source = self.extractfile(tarinfo)
+ target = bltn_open(targetpath, "wb")
+ copyfileobj(source, target)
+ source.close()
+ target.close()
+
+ def makeunknown(self, tarinfo, targetpath):
+ """Make a file from a TarInfo object with an unknown type
+ at targetpath.
+ """
+ self.makefile(tarinfo, targetpath)
+ self._dbg(1, "tarfile: Unknown file type %r, " \
+ "extracted as regular file." % tarinfo.type)
+
+ def makefifo(self, tarinfo, targetpath):
+ """Make a fifo called targetpath.
+ """
+ if hasattr(os, "mkfifo"):
+ os.mkfifo(targetpath)
+ else:
+ raise ExtractError("fifo not supported by system")
+
+ def makedev(self, tarinfo, targetpath):
+ """Make a character or block device called targetpath.
+ """
+ if not hasattr(os, "mknod") or not hasattr(os, "makedev"):
+ raise ExtractError("special devices not supported by system")
+
+ mode = tarinfo.mode
+ if tarinfo.isblk():
+ mode |= stat.S_IFBLK
+ else:
+ mode |= stat.S_IFCHR
+
+ os.mknod(targetpath, mode,
+ os.makedev(tarinfo.devmajor, tarinfo.devminor))
+
+ def makelink(self, tarinfo, targetpath):
+ """Make a (symbolic) link called targetpath. If it cannot be created
+ (platform limitation), we try to make a copy of the referenced file
+ instead of a link.
+ """
+ if hasattr(os, "symlink") and hasattr(os, "link"):
+ # For systems that support symbolic and hard links.
+ if tarinfo.issym():
+ if os.path.lexists(targetpath):
+ os.unlink(targetpath)
+ os.symlink(tarinfo.linkname, targetpath)
+ else:
+ # See extract().
+ if os.path.exists(tarinfo._link_target):
+ if os.path.lexists(targetpath):
+ os.unlink(targetpath)
+ os.link(tarinfo._link_target, targetpath)
+ else:
+ self._extract_member(self._find_link_target(tarinfo), targetpath)
+ else:
+ try:
+ self._extract_member(self._find_link_target(tarinfo), targetpath)
+ except KeyError:
+ raise ExtractError("unable to resolve link inside archive")
+
+ def chown(self, tarinfo, targetpath):
+ """Set owner of targetpath according to tarinfo.
+ """
+ if pwd and hasattr(os, "geteuid") and os.geteuid() == 0:
+ # We have to be root to do so.
+ try:
+ g = grp.getgrnam(tarinfo.gname)[2]
+ except KeyError:
+ try:
+ g = grp.getgrgid(tarinfo.gid)[2]
+ except KeyError:
+ g = os.getgid()
+ try:
+ u = pwd.getpwnam(tarinfo.uname)[2]
+ except KeyError:
+ try:
+ u = pwd.getpwuid(tarinfo.uid)[2]
+ except KeyError:
+ u = os.getuid()
+ try:
+ if tarinfo.issym() and hasattr(os, "lchown"):
+ os.lchown(targetpath, u, g)
+ else:
+ if sys.platform != "os2emx":
+ os.chown(targetpath, u, g)
+ except EnvironmentError, e:
+ raise ExtractError("could not change owner")
+
+ def chmod(self, tarinfo, targetpath):
+ """Set file permissions of targetpath according to tarinfo.
+ """
+ if hasattr(os, 'chmod'):
+ try:
+ os.chmod(targetpath, tarinfo.mode)
+ except EnvironmentError, e:
+ raise ExtractError("could not change mode")
+
+ def utime(self, tarinfo, targetpath):
+ """Set modification time of targetpath according to tarinfo.
+ """
+ if not hasattr(os, 'utime'):
+ return
+ try:
+ os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
+ except EnvironmentError, e:
+ raise ExtractError("could not change modification time")
+
+ #--------------------------------------------------------------------------
+ def next(self):
+ """Return the next member of the archive as a TarInfo object, when
+ TarFile is opened for reading. Return None if there is no more
+ available.
+ """
+ self._check("ra")
+ if self.firstmember is not None:
+ m = self.firstmember
+ self.firstmember = None
+ return m
+
+ # Read the next block.
+ self.fileobj.seek(self.offset)
+ tarinfo = None
+ while True:
+ try:
+ tarinfo = self.tarinfo.fromtarfile(self)
+ except EOFHeaderError, e:
+ if self.ignore_zeros:
+ self._dbg(2, "0x%X: %s" % (self.offset, e))
+ self.offset += BLOCKSIZE
+ continue
+ except InvalidHeaderError, e:
+ if self.ignore_zeros:
+ self._dbg(2, "0x%X: %s" % (self.offset, e))
+ self.offset += BLOCKSIZE
+ continue
+ elif self.offset == 0:
+ raise ReadError(str(e))
+ except EmptyHeaderError:
+ if self.offset == 0:
+ raise ReadError("empty file")
+ except TruncatedHeaderError, e:
+ if self.offset == 0:
+ raise ReadError(str(e))
+ except SubsequentHeaderError, e:
+ raise ReadError(str(e))
+ break
+
+ if tarinfo is not None:
+ self.members.append(tarinfo)
+ else:
+ self._loaded = True
+
+ return tarinfo
+
+ #--------------------------------------------------------------------------
+ # Little helper methods:
+
+ def _getmember(self, name, tarinfo=None, normalize=False):
+ """Find an archive member by name from bottom to top.
+ If tarinfo is given, it is used as the starting point.
+ """
+ # Ensure that all members have been loaded.
+ members = self.getmembers()
+
+ # Limit the member search list up to tarinfo.
+ if tarinfo is not None:
+ members = members[:members.index(tarinfo)]
+
+ if normalize:
+ name = os.path.normpath(name)
+
+ for member in reversed(members):
+ if normalize:
+ member_name = os.path.normpath(member.name)
+ else:
+ member_name = member.name
+
+ if name == member_name:
+ return member
+
+ def _load(self):
+ """Read through the entire archive file and look for readable
+ members.
+ """
+ while True:
+ tarinfo = self.next()
+ if tarinfo is None:
+ break
+ self._loaded = True
+
+ def _check(self, mode=None):
+ """Check if TarFile is still open, and if the operation's mode
+ corresponds to TarFile's mode.
+ """
+ if self.closed:
+ raise IOError("%s is closed" % self.__class__.__name__)
+ if mode is not None and self.mode not in mode:
+ raise IOError("bad operation for mode %r" % self.mode)
+
+ def _find_link_target(self, tarinfo):
+ """Find the target member of a symlink or hardlink member in the
+ archive.
+ """
+ if tarinfo.issym():
+ # Always search the entire archive.
+ linkname = os.path.dirname(tarinfo.name) + "/" + tarinfo.linkname
+ limit = None
+ else:
+ # Search the archive before the link, because a hard link is
+ # just a reference to an already archived file.
+ linkname = tarinfo.linkname
+ limit = tarinfo
+
+ member = self._getmember(linkname, tarinfo=limit, normalize=True)
+ if member is None:
+ raise KeyError("linkname %r not found" % linkname)
+ return member
+
+ def __iter__(self):
+ """Provide an iterator object.
+ """
+ if self._loaded:
+ return iter(self.members)
+ else:
+ return TarIter(self)
+
+ def _dbg(self, level, msg):
+ """Write debugging output to sys.stderr.
+ """
+ if level <= self.debug:
+ print >> sys.stderr, msg
+
+ def __enter__(self):
+ self._check()
+ return self
+
+ def __exit__(self, type, value, traceback):
+ if type is None:
+ self.close()
+ else:
+ # An exception occurred. We must not call close() because
+ # it would try to write end-of-archive blocks and padding.
+ if not self._extfileobj:
+ self.fileobj.close()
+ self.closed = True
+# class TarFile
+
+class TarIter:
+ """Iterator Class.
+
+ for tarinfo in TarFile(...):
+ suite...
+ """
+
+ def __init__(self, tarfile):
+ """Construct a TarIter object.
+ """
+ self.tarfile = tarfile
+ self.index = 0
+ def __iter__(self):
+ """Return iterator object.
+ """
+ return self
+ def next(self):
+ """Return the next item using TarFile's next() method.
+ When all members have been read, set TarFile as _loaded.
+ """
+ # Fix for SF #1100429: Under rare circumstances it can
+ # happen that getmembers() is called during iteration,
+ # which will cause TarIter to stop prematurely.
+ if not self.tarfile._loaded:
+ tarinfo = self.tarfile.next()
+ if not tarinfo:
+ self.tarfile._loaded = True
+ raise StopIteration
+ else:
+ try:
+ tarinfo = self.tarfile.members[self.index]
+ except IndexError:
+ raise StopIteration
+ self.index += 1
+ return tarinfo
+
+# Helper classes for sparse file support
+class _section:
+ """Base class for _data and _hole.
+ """
+ def __init__(self, offset, size):
+ self.offset = offset
+ self.size = size
+ def __contains__(self, offset):
+ return self.offset <= offset < self.offset + self.size
+
+class _data(_section):
+ """Represent a data section in a sparse file.
+ """
+ def __init__(self, offset, size, realpos):
+ _section.__init__(self, offset, size)
+ self.realpos = realpos
+
+class _hole(_section):
+ """Represent a hole section in a sparse file.
+ """
+ pass
+
+class _ringbuffer(list):
+ """Ringbuffer class which increases performance
+ over a regular list.
+ """
+ def __init__(self):
+ self.idx = 0
+ def find(self, offset):
+ idx = self.idx
+ while True:
+ item = self[idx]
+ if offset in item:
+ break
+ idx += 1
+ if idx == len(self):
+ idx = 0
+ if idx == self.idx:
+ # End of File
+ return None
+ self.idx = idx
+ return item
+
+#---------------------------------------------
+# zipfile compatible TarFile class
+#---------------------------------------------
+TAR_PLAIN = 0 # zipfile.ZIP_STORED
+TAR_GZIPPED = 8 # zipfile.ZIP_DEFLATED
+class TarFileCompat:
+ """TarFile class compatible with standard module zipfile's
+ ZipFile class.
+ """
+ def __init__(self, file, mode="r", compression=TAR_PLAIN):
+ from warnings import warnpy3k
+ warnpy3k("the TarFileCompat class has been removed in Python 3.0",
+ stacklevel=2)
+ if compression == TAR_PLAIN:
+ self.tarfile = TarFile.taropen(file, mode)
+ elif compression == TAR_GZIPPED:
+ self.tarfile = TarFile.gzopen(file, mode)
+ else:
+ raise ValueError("unknown compression constant")
+ if mode[0:1] == "r":
+ members = self.tarfile.getmembers()
+ for m in members:
+ m.filename = m.name
+ m.file_size = m.size
+ m.date_time = time.gmtime(m.mtime)[:6]
+ def namelist(self):
+ return map(lambda m: m.name, self.infolist())
+ def infolist(self):
+ return filter(lambda m: m.type in REGULAR_TYPES,
+ self.tarfile.getmembers())
+ def printdir(self):
+ self.tarfile.list()
+ def testzip(self):
+ return
+ def getinfo(self, name):
+ return self.tarfile.getmember(name)
+ def read(self, name):
+ return self.tarfile.extractfile(self.tarfile.getmember(name)).read()
+ def write(self, filename, arcname=None, compress_type=None):
+ self.tarfile.add(filename, arcname)
+ def writestr(self, zinfo, bytes):
+ try:
+ from cStringIO import StringIO
+ except ImportError:
+ from StringIO import StringIO
+ import calendar
+ tinfo = TarInfo(zinfo.filename)
+ tinfo.size = len(bytes)
+ tinfo.mtime = calendar.timegm(zinfo.date_time)
+ self.tarfile.addfile(tinfo, StringIO(bytes))
+ def close(self):
+ self.tarfile.close()
+#class TarFileCompat
+
+#--------------------
+# exported functions
+#--------------------
+def is_tarfile(name):
+ """Return True if name points to a tar archive that we
+ are able to handle, else return False.
+ """
+ try:
+ t = open(name)
+ t.close()
+ return True
+ except TarError:
+ return False
+
+bltn_open = open
+open = TarFile.open
--
cgit v1.3