summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/__init__.py1
-rw-r--r--lib/archive.py289
-rw-r--r--lib/audio.py98
-rw-r--r--lib/bencode/__init__.py1
-rw-r--r--lib/bencode/bencode.py142
-rw-r--r--lib/hachoir_editor/__init__.py8
-rw-r--r--lib/hachoir_editor/field.py69
-rw-r--r--lib/hachoir_editor/fieldset.py347
-rw-r--r--lib/hachoir_editor/typed_field.py253
-rw-r--r--lib/images.py37
-rw-r--r--lib/mat.py162
-rw-r--r--lib/misc.py63
-rw-r--r--lib/office.py257
-rw-r--r--lib/parser.py104
-rw-r--r--lib/pdfrw/__init__.py14
-rw-r--r--lib/pdfrw/pdfcompress.py57
-rw-r--r--lib/pdfrw/pdfobjects.py183
-rw-r--r--lib/pdfrw/pdfreader.py213
-rw-r--r--lib/pdfrw/pdftokens.py249
-rw-r--r--lib/pdfrw/pdfwriter.py234
-rw-r--r--lib/tarfile/__init__.py1
-rwxr-xr-xlib/tarfile/tarfile.py2594
22 files changed, 0 insertions, 5376 deletions
diff --git a/lib/__init__.py b/lib/__init__.py
deleted file mode 100644
index 8b13789..0000000
--- a/lib/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
1
diff --git a/lib/archive.py b/lib/archive.py
deleted file mode 100644
index 77db71c..0000000
--- a/lib/archive.py
+++ /dev/null
@@ -1,289 +0,0 @@
1'''
2 Take care of archives formats
3'''
4
5import zipfile
6import shutil
7import os
8import logging
9import tempfile
10
11import parser
12import mat
13from tarfile import tarfile
14
15
16class GenericArchiveStripper(parser.GenericParser):
17 '''
18 Represent a generic archive
19 '''
20 def __init__(self, filename, parser, mime, backup, add2archive):
21 super(GenericArchiveStripper, self).__init__(filename, parser, mime,
22 backup, add2archive)
23 self.compression = ''
24 self.add2archive = add2archive
25 self.tempdir = tempfile.mkdtemp()
26
27 def __del__(self):
28 '''
29 Remove the files inside the temp dir,
30 then remove the temp dir
31 '''
32 for root, dirs, files in os.walk(self.tempdir):
33 for item in files:
34 path_file = os.path.join(root, item)
35 mat.secure_remove(path_file)
36 shutil.rmtree(self.tempdir)
37
38 def remove_all(self):
39 '''
40 Call _remove_all() with in argument : "normal"
41 '''
42 self._remove_all('normal')
43
44 def remove_all_ugly(self):
45 '''
46 call remove_all() with in argument : "ugly"
47 '''
48 self._remove_all('ugly')
49
50 def _remove_all(self, method):
51 '''
52 Remove all meta, normal way if method is "normal",
53 else, use the ugly way (with possible data loss)
54 '''
55 raise NotImplementedError
56
57
58class ZipStripper(GenericArchiveStripper):
59 '''
60 Represent a zip file
61 '''
62 def is_file_clean(self, fileinfo):
63 '''
64 Check if a ZipInfo object is clean of metadatas added
65 by zip itself, independently of the corresponding file metadatas
66 '''
67 if fileinfo.comment is not '':
68 return False
69 elif fileinfo.date_time is not 0:
70 return False
71 elif fileinfo.create_system is not 0:
72 return False
73 elif fileinfo.create_version is not 0:
74 return False
75 else:
76 return True
77
78 def is_clean(self):
79 '''
80 Check if the given file is clean from harmful metadata
81 '''
82 zipin = zipfile.ZipFile(self.filename, 'r')
83 if zipin.comment != '':
84 logging.debug('%s has a comment' % self.filename)
85 return False
86 for item in zipin.infolist():
87 #I have not found a way to remove the crap added by zipfile :/
88 #if not self.is_file_clean(item):
89 # logging.debug('%s from %s has compromizing zipinfo' %
90 # (item.filename, self.filename))
91 # return False
92 zipin.extract(item, self.tempdir)
93 name = os.path.join(self.tempdir, item.filename)
94 if os.path.isfile(name):
95 try:
96 cfile = mat.create_class_file(name, False,
97 self.add2archive)
98 if not cfile.is_clean():
99 return False
100 except:
101 #best solution I have found
102 logging.info('%s\'s fileformat is not supported, or is a \
103harmless format' % item.filename)
104 _, ext = os.path.splitext(name)
105 bname = os.path.basename(item.filename)
106 if ext not in parser.NOMETA:
107 if bname != 'mimetype' and bname != '.rels':
108 return False
109 zipin.close()
110 return True
111
112 def get_meta(self):
113 '''
114 Return all the metadata of a ZipFile (don't return metadatas
115 of contained files : should it ?)
116 '''
117 zipin = zipfile.ZipFile(self.filename, 'r')
118 metadata = {}
119 for field in zipin.infolist():
120 zipmeta = {}
121 zipmeta['comment'] = field.comment
122 zipmeta['modified'] = field.date_time
123 zipmeta['system'] = field.create_system
124 zipmeta['zip_version'] = field.create_version
125 metadata[field.filename] = zipmeta
126 metadata["%s comment" % self.filename] = zipin.comment
127 zipin.close()
128 return metadata
129
130 def _remove_all(self, method):
131 '''
132 So far, the zipfile module does not allow to write a ZipInfo
133 object into a zipfile (and it's a shame !) : so data added
134 by zipfile itself could not be removed. It's a big concern.
135 Is shiping a patched version of zipfile.py a good idea ?
136 '''
137 zipin = zipfile.ZipFile(self.filename, 'r')
138 zipout = zipfile.ZipFile(self.output, 'w', allowZip64=True)
139 for item in zipin.infolist():
140 zipin.extract(item, self.tempdir)
141 name = os.path.join(self.tempdir, item.filename)
142 if os.path.isfile(name):
143 try:
144 cfile = mat.create_class_file(name, False,
145 self.add2archive)
146 if method is 'normal':
147 cfile.remove_all()
148 else:
149 cfile.remove_all_ugly()
150 logging.debug('Processing %s from %s' % (item.filename,
151 self.filename))
152 zipout.write(name, item.filename)
153 except:
154 logging.info('%s\'s format is not supported or harmless' %
155 item.filename)
156 _, ext = os.path.splitext(name)
157 if self.add2archive or ext in parser.NOMETA:
158 zipout.write(name, item.filename)
159 zipout.comment = ''
160 zipin.close()
161 zipout.close()
162 logging.info('%s treated' % self.filename)
163 self.do_backup()
164
165
166class TarStripper(GenericArchiveStripper):
167 '''
168 Represent a tarfile archive
169 '''
170 def _remove(self, current_file):
171 '''
172 remove the meta added by tar itself to the file
173 '''
174 current_file.mtime = 0
175 current_file.uid = 0
176 current_file.gid = 0
177 current_file.uname = ''
178 current_file.gname = ''
179 return current_file
180
181 def _remove_all(self, method):
182 tarin = tarfile.open(self.filename, 'r' + self.compression)
183 tarout = tarfile.open(self.output, 'w' + self.compression)
184 for item in tarin.getmembers():
185 tarin.extract(item, self.tempdir)
186 name = os.path.join(self.tempdir, item.name)
187 if item.type is '0': # is item a regular file ?
188 #no backup file
189 try:
190 cfile = mat.create_class_file(name, False,
191 self.add2archive)
192 if method is 'normal':
193 cfile.remove_all()
194 else:
195 cfile.remove_all_ugly()
196 tarout.add(name, item.name, filter=self._remove)
197 except:
198 logging.info('%s\' format is not supported or harmless' %
199 item.name)
200 _, ext = os.path.splitext(name)
201 if self.add2archive or ext in parser.NOMETA:
202 tarout.add(name, item.name, filter=self._remove)
203 tarin.close()
204 tarout.close()
205 self.do_backup()
206
207 def is_file_clean(self, current_file):
208 '''
209 Check metadatas added by tar
210 '''
211 if current_file.mtime is not 0:
212 return False
213 elif current_file.uid is not 0:
214 return False
215 elif current_file.gid is not 0:
216 return False
217 elif current_file.uname is not '':
218 return False
219 elif current_file.gname is not '':
220 return False
221 else:
222 return True
223
224 def is_clean(self):
225 '''
226 Check if the file is clean from harmful metadatas
227 '''
228 tarin = tarfile.open(self.filename, 'r' + self.compression)
229 for item in tarin.getmembers():
230 if not self.is_file_clean(item):
231 tarin.close()
232 return False
233 tarin.extract(item, self.tempdir)
234 name = os.path.join(self.tempdir, item.name)
235 if item.type is '0': # is item a regular file ?
236 try:
237 class_file = mat.create_class_file(name,
238 False, self.add2archive) # no backup file
239 if not class_file.is_clean():
240 tarin.close()
241 return False
242 except:
243 logging.error('%s\'s foramt is not supported or harmless' %
244 item.filename)
245 _, ext = os.path.splitext(name)
246 if ext not in parser.NOMETA:
247 tarin.close()
248 return False
249 tarin.close()
250 return True
251
252 def get_meta(self):
253 '''
254 Return a dict with all the meta of the file
255 '''
256 tarin = tarfile.open(self.filename, 'r' + self.compression)
257 metadata = {}
258 for current_file in tarin.getmembers():
259 if current_file.type is '0':
260 if not self.is_file_clean(current_file): # if there is meta
261 current_meta = {}
262 current_meta['mtime'] = current_file.mtime
263 current_meta['uid'] = current_file.uid
264 current_meta['gid'] = current_file.gid
265 current_meta['uname'] = current_file.uname
266 current_meta['gname'] = current_file.gname
267 metadata[current_file.name] = current_meta
268 tarin.close()
269 return metadata
270
271
272class GzipStripper(TarStripper):
273 '''
274 Represent a tar.gz archive
275 '''
276 def __init__(self, filename, parser, mime, backup, add2archive):
277 super(GzipStripper, self).__init__(filename, parser, mime, backup,
278 add2archive)
279 self.compression = ':gz'
280
281
282class Bzip2Stripper(TarStripper):
283 '''
284 Represents a tar.bz2 archive
285 '''
286 def __init__(self, filename, parser, mime, backup, add2archive):
287 super(Bzip2Stripper, self).__init__(filename, parser, mime, backup,
288 add2archive)
289 self.compression = ':bz2'
diff --git a/lib/audio.py b/lib/audio.py
deleted file mode 100644
index 18daa7e..0000000
--- a/lib/audio.py
+++ /dev/null
@@ -1,98 +0,0 @@
1'''
2 Care about audio fileformat
3'''
4try:
5 from mutagen.flac import FLAC
6 from mutagen.oggvorbis import OggVorbis
7except ImportError:
8 pass
9
10
11import parser
12import shutil
13
14
15class MpegAudioStripper(parser.GenericParser):
16 '''
17 Represent mpeg audio file (mp3, ...)
18 '''
19 def _should_remove(self, field):
20 if field.name in ("id3v1", "id3v2"):
21 return True
22 else:
23 return False
24
25
26class OggStripper(parser.GenericParser):
27 '''
28 Represent an ogg vorbis file
29 '''
30 def remove_all(self):
31 if self.backup is True:
32 shutil.copy2(self.filename, self.output)
33 self.filename = self.output
34
35 mfile = OggVorbis(self.filename)
36 mfile.delete()
37 mfile.save()
38
39 def is_clean(self):
40 '''
41 Check if the "metadata" block is present in the file
42 '''
43 mfile = OggVorbis(self.filename)
44 if mfile.tags == []:
45 return True
46 else:
47 return False
48
49 def get_meta(self):
50 '''
51 Return the content of the metadata block if present
52 '''
53 metadata = {}
54 mfile = OggVorbis(self.filename)
55 for key, value in mfile.tags:
56 metadata[key] = value
57 return metadata
58
59
60class FlacStripper(parser.GenericParser):
61 '''
62 Represent a Flac audio file
63 '''
64 def remove_all(self):
65 '''
66 Remove the "metadata" block from the file
67 '''
68 if self.backup is True:
69 shutil.copy2(self.filename, self.output)
70 self.filename = self.output
71
72 mfile = FLAC(self.filename)
73 mfile.delete()
74 mfile.clear_pictures()
75 mfile.save()
76
77 def is_clean(self):
78 '''
79 Check if the "metadata" block is present in the file
80 '''
81 mfile = FLAC(self.filename)
82 if mfile.tags is None and mfile.pictures == []:
83 return True
84 else:
85 return False
86
87 def get_meta(self):
88 '''
89 Return the content of the metadata block if present
90 '''
91 metadata = {}
92 mfile = FLAC(self.filename)
93 if mfile.tags is not None:
94 for key, value in mfile.tags:
95 metadata[key] = value
96 if mfile.pictures != []:
97 metadata['picture :'] = 'yes'
98 return metadata
diff --git a/lib/bencode/__init__.py b/lib/bencode/__init__.py
deleted file mode 100644
index 8b13789..0000000
--- a/lib/bencode/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
1
diff --git a/lib/bencode/bencode.py b/lib/bencode/bencode.py
deleted file mode 100644
index 4acf788..0000000
--- a/lib/bencode/bencode.py
+++ /dev/null
@@ -1,142 +0,0 @@
1# The contents of this file are subject to the BitTorrent Open Source License
2# Version 1.1 (the License). You may not copy or use this file, in either
3# source code or executable form, except in compliance with the License. You
4# may obtain a copy of the License at http://www.bittorrent.com/license/.
5#
6# Software distributed under the License is distributed on an AS IS basis,
7# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
8# for the specific language governing rights and limitations under the
9# License.
10
11# Written by Petru Paler
12# Modified by Julien (jvoisin) Voisin
13
14'''
15 A quick (and also nice) lib to bencode/bdecode torrent files
16'''
17
18
19import types
20
21
22class BTFailure(Exception):
23 '''Custom Exception'''
24 pass
25
26
27class Bencached(object):
28 '''Custom type : cached string'''
29 __slots__ = ['bencoded']
30
31 def __init__(self, string):
32 self.bencoded = string
33
34
35def decode_int(x, f):
36 '''decode an int'''
37 f += 1
38 newf = x.index('e', f)
39 n = int(x[f:newf])
40 if x[f] == '-':
41 if x[f + 1] == '0':
42 raise ValueError
43 elif x[f] == '0' and newf != f + 1:
44 raise ValueError
45 return (n, newf + 1)
46
47
48def decode_string(x, f):
49 '''decode a string'''
50 colon = x.index(':', f)
51 n = int(x[f:colon])
52 if x[f] == '0' and colon != f + 1:
53 raise ValueError
54 colon += 1
55 return (x[colon:colon + n], colon + n)
56
57
58def decode_list(x, f):
59 '''decode a list'''
60 result = []
61 f += 1
62 while x[f] != 'e':
63 v, f = DECODE_FUNC[x[f]](x, f)
64 result.append(v)
65 return (result, f + 1)
66
67
68def decode_dict(x, f):
69 '''decode a dict'''
70 result = {}
71 f += 1
72 while x[f] != 'e':
73 k, f = decode_string(x, f)
74 result[k], f = DECODE_FUNC[x[f]](x, f)
75 return (result, f + 1)
76
77
78def encode_bool(x, r):
79 '''bencode a boolean'''
80 if x:
81 encode_int(1, r)
82 else:
83 encode_int(0, r)
84
85
86def encode_int(x, r):
87 '''bencode an integer/float'''
88 r.extend(('i', str(x), 'e'))
89
90
91def encode_list(x, r):
92 '''bencode a list/tuple'''
93 r.append('l')
94 [ENCODE_FUNC[type(item)](item, r) for item in x]
95 r.append('e')
96
97
98def encode_dict(x, result):
99 '''bencode a dict'''
100 result.append('d')
101 ilist = x.items()
102 ilist.sort()
103 for k, v in ilist:
104 result.extend((str(len(k)), ':', k))
105 ENCODE_FUNC[type(v)](v, result)
106 result.append('e')
107
108
109DECODE_FUNC = {}
110DECODE_FUNC.update(dict([(str(x), decode_string) for x in xrange(9)]))
111DECODE_FUNC['l'] = decode_list
112DECODE_FUNC['d'] = decode_dict
113DECODE_FUNC['i'] = decode_int
114
115
116ENCODE_FUNC = {}
117ENCODE_FUNC[Bencached] = lambda x, r: r.append(x.bencoded)
118ENCODE_FUNC[types.IntType] = encode_int
119ENCODE_FUNC[types.LongType] = encode_int
120ENCODE_FUNC[types.StringType] = lambda x, r: r.extend((str(len(x)), ':', x))
121ENCODE_FUNC[types.ListType] = encode_list
122ENCODE_FUNC[types.TupleType] = encode_list
123ENCODE_FUNC[types.DictType] = encode_dict
124ENCODE_FUNC[types.BooleanType] = encode_bool
125
126
127def bencode(string):
128 '''bencode $string'''
129 table = []
130 ENCODE_FUNC[type(string)](string, table)
131 return ''.join(table)
132
133
134def bdecode(string):
135 '''decode $string'''
136 try:
137 result, lenght = DECODE_FUNC[string[0]](string, 0)
138 except (IndexError, KeyError, ValueError):
139 raise BTFailure('Not a valid bencoded string')
140 if lenght != len(string):
141 raise BTFailure('Invalid bencoded value (data after valid prefix)')
142 return result
diff --git a/lib/hachoir_editor/__init__.py b/lib/hachoir_editor/__init__.py
deleted file mode 100644
index 1835676..0000000
--- a/lib/hachoir_editor/__init__.py
+++ /dev/null
@@ -1,8 +0,0 @@
1from field import (
2 EditorError, FakeField)
3from typed_field import (
4 EditableField, EditableBits, EditableBytes,
5 EditableInteger, EditableString,
6 createEditableField)
7from fieldset import EditableFieldSet, NewFieldSet, createEditor
8
diff --git a/lib/hachoir_editor/field.py b/lib/hachoir_editor/field.py
deleted file mode 100644
index 6b1efe3..0000000
--- a/lib/hachoir_editor/field.py
+++ /dev/null
@@ -1,69 +0,0 @@
1from hachoir_core.error import HachoirError
2from hachoir_core.field import joinPath, MissingField
3
4class EditorError(HachoirError):
5 pass
6
7class FakeField(object):
8 """
9 This class have API looks similar to Field API, but objects don't contain
10 any value: all values are _computed_ by parent methods.
11
12 Example: FakeField(editor, "abc").size calls editor._getFieldSize("abc").
13 """
14 is_field_set = False
15
16 def __init__(self, parent, name):
17 self._parent = parent
18 self._name = name
19
20 def _getPath(self):
21 return joinPath(self._parent.path, self._name)
22 path = property(_getPath)
23
24 def _getName(self):
25 return self._name
26 name = property(_getName)
27
28 def _getAddress(self):
29 return self._parent._getFieldAddress(self._name)
30 address = property(_getAddress)
31
32 def _getSize(self):
33 return self._parent.input[self._name].size
34 size = property(_getSize)
35
36 def _getValue(self):
37 return self._parent.input[self._name].value
38 value = property(_getValue)
39
40 def createDisplay(self):
41 # TODO: Returns new value if field is altered
42 return self._parent.input[self._name].display
43 display = property(createDisplay)
44
45 def _getParent(self):
46 return self._parent
47 parent = property(_getParent)
48
49 def hasValue(self):
50 return self._parent.input[self._name].hasValue()
51
52 def __getitem__(self, key):
53 # TODO: Implement this function!
54 raise MissingField(self, key)
55
56 def _isAltered(self):
57 return False
58 is_altered = property(_isAltered)
59
60 def writeInto(self, output):
61 size = self.size
62 addr = self._parent._getFieldInputAddress(self._name)
63 input = self._parent.input
64 stream = input.stream
65 if size % 8:
66 output.copyBitsFrom(stream, addr, size, input.endian)
67 else:
68 output.copyBytesFrom(stream, addr, size//8)
69
diff --git a/lib/hachoir_editor/fieldset.py b/lib/hachoir_editor/fieldset.py
deleted file mode 100644
index cbc12f9..0000000
--- a/lib/hachoir_editor/fieldset.py
+++ /dev/null
@@ -1,347 +0,0 @@
1from hachoir_core.dict import UniqKeyError
2from hachoir_core.field import MissingField, Float32, Float64, FakeArray
3from hachoir_core.compatibility import any
4from hachoir_core.i18n import _
5from typed_field import createEditableField
6from field import EditorError
7from collections import deque # Python 2.4
8import weakref # Python 2.1
9import struct
10
11class EditableFieldSet(object):
12 MAX_SIZE = (1 << 40) # Arbitrary limit to catch errors
13 is_field_set = True
14
15 def __init__(self, parent, fieldset):
16 self._parent = parent
17 self.input = fieldset # original FieldSet
18 self._fields = {} # cache of editable fields
19 self._deleted = set() # Names of deleted fields
20 self._inserted = {} # Inserted field (name => list of field,
21 # where name is the name after)
22
23 def array(self, key):
24 # FIXME: Use cache?
25 return FakeArray(self, key)
26
27 def _getParent(self):
28 return self._parent
29 parent = property(_getParent)
30
31 def _isAltered(self):
32 if self._inserted:
33 return True
34 if self._deleted:
35 return True
36 return any(field.is_altered for field in self._fields.itervalues())
37 is_altered = property(_isAltered)
38
39 def reset(self):
40 """
41 Reset the field set and the input field set.
42 """
43 for key, field in self._fields.iteritems():
44 if not field.is_altered:
45 del self._fields[key]
46 self.input.reset()
47
48 def __len__(self):
49 return len(self.input) \
50 - len(self._deleted) \
51 + sum( len(new) for new in self._inserted.itervalues() )
52
53 def __iter__(self):
54 for field in self.input:
55 name = field.name
56 if name in self._inserted:
57 for newfield in self._inserted[name]:
58 yield weakref.proxy(newfield)
59 if name not in self._deleted:
60 yield self[name]
61 if None in self._inserted:
62 for newfield in self._inserted[None]:
63 yield weakref.proxy(newfield)
64
65 def insertBefore(self, name, *new_fields):
66 self._insert(name, new_fields, False)
67
68 def insertAfter(self, name, *new_fields):
69 self._insert(name, new_fields, True)
70
71 def insert(self, *new_fields):
72 self._insert(None, new_fields, True)
73
74 def _insert(self, key, new_fields, next):
75 """
76 key is the name of the field before which new_fields
77 will be inserted. If next is True, the fields will be inserted
78 _after_ this field.
79 """
80 # Set unique field name
81 for field in new_fields:
82 if field._name.endswith("[]"):
83 self.input.setUniqueFieldName(field)
84
85 # Check that there is no duplicate in inserted fields
86 new_names = list(field.name for field in new_fields)
87 names_set = set(new_names)
88 if len(names_set) != len(new_fields):
89 duplicates = (name for name in names_set if 1 < new_names.count(name))
90 raise UniqKeyError(_("Duplicates in inserted fields: %s") % ", ".join(duplicates))
91
92 # Check that field names are not in input
93 if self.input: # Write special version for NewFieldSet?
94 for name in new_names:
95 if name in self.input and name not in self._deleted:
96 raise UniqKeyError(_("Field name '%s' already exists") % name)
97
98 # Check that field names are not in inserted fields
99 for fields in self._inserted.itervalues():
100 for field in fields:
101 if field.name in new_names:
102 raise UniqKeyError(_("Field name '%s' already exists") % field.name)
103
104 # Input have already inserted field?
105 if key in self._inserted:
106 if next:
107 self._inserted[key].extend( reversed(new_fields) )
108 else:
109 self._inserted[key].extendleft( reversed(new_fields) )
110 return
111
112 # Whould like to insert in inserted fields?
113 if key:
114 for fields in self._inserted.itervalues():
115 names = [item.name for item in fields]
116 try:
117 pos = names.index(key)
118 except ValueError:
119 continue
120 if 0 <= pos:
121 if next:
122 pos += 1
123 fields.rotate(-pos)
124 fields.extendleft( reversed(new_fields) )
125 fields.rotate(pos)
126 return
127
128 # Get next field. Use None if we are at the end.
129 if next:
130 index = self.input[key].index + 1
131 try:
132 key = self.input[index].name
133 except IndexError:
134 key = None
135
136 # Check that field names are not in input
137 if key not in self.input:
138 raise MissingField(self, key)
139
140 # Insert in original input
141 self._inserted[key]= deque(new_fields)
142
143 def _getDescription(self):
144 return self.input.description
145 description = property(_getDescription)
146
147 def _getStream(self):
148 # FIXME: This property is maybe a bad idea since address may be differents
149 return self.input.stream
150 stream = property(_getStream)
151
152 def _getName(self):
153 return self.input.name
154 name = property(_getName)
155
156 def _getEndian(self):
157 return self.input.endian
158 endian = property(_getEndian)
159
160 def _getAddress(self):
161 if self._parent:
162 return self._parent._getFieldAddress(self.name)
163 else:
164 return 0
165 address = property(_getAddress)
166
167 def _getAbsoluteAddress(self):
168 address = self.address
169 current = self._parent
170 while current:
171 address += current.address
172 current = current._parent
173 return address
174 absolute_address = property(_getAbsoluteAddress)
175
176 def hasValue(self):
177 return False
178# return self._parent.input[self.name].hasValue()
179
180 def _getSize(self):
181 if self.is_altered:
182 return sum(field.size for field in self)
183 else:
184 return self.input.size
185 size = property(_getSize)
186
187 def _getPath(self):
188 return self.input.path
189 path = property(_getPath)
190
191 def _getOriginalField(self, name):
192 assert name in self.input
193 return self.input[name]
194
195 def _getFieldInputAddress(self, name):
196 """
197 Absolute address of a field from the input field set.
198 """
199 assert name in self.input
200 return self.input[name].absolute_address
201
202 def _getFieldAddress(self, name):
203 """
204 Compute relative address of a field. The operation takes care of
205 deleted and resized fields.
206 """
207 #assert name not in self._deleted
208 addr = 0
209 for field in self:
210 if field.name == name:
211 return addr
212 addr += field.size
213 raise MissingField(self, name)
214
215 def _getItemByPath(self, path):
216 if not path[0]:
217 path = path[1:]
218 field = self
219 for name in path:
220 field = field[name]
221 return field
222
223 def __contains__(self, name):
224 try:
225 field = self[name]
226 return (field is not None)
227 except MissingField:
228 return False
229
230 def __getitem__(self, key):
231 """
232 Create a weak reference to an editable field (EditableField) for the
233 field with specified name. If the field is removed later, using the
234 editable field will raise a weakref.ReferenceError exception.
235
236 May raise a MissingField error if the field doesn't exist in original
237 field set or it has been deleted.
238 """
239 if "/" in key:
240 return self._getItemByPath(key.split("/"))
241 if isinstance(key, (int, long)):
242 raise EditorError("Integer index are not supported")
243
244 if (key in self._deleted) or (key not in self.input):
245 raise MissingField(self, key)
246 if key not in self._fields:
247 field = self.input[key]
248 if field.is_field_set:
249 self._fields[key] = createEditableFieldSet(self, field)
250 else:
251 self._fields[key] = createEditableField(self, field)
252 return weakref.proxy(self._fields[key])
253
254 def __delitem__(self, name):
255 """
256 Remove a field from the field set. May raise an MissingField exception
257 if the field has already been deleted.
258 """
259 if name in self._deleted:
260 raise MissingField(self, name)
261 self._deleted.add(name)
262 if name in self._fields:
263 del self._fields[name]
264
265 def writeInto(self, output):
266 """
267 Write the content if this field set into the output stream
268 (OutputStream).
269 """
270 if not self.is_altered:
271 # Not altered: just copy bits/bytes
272 input = self.input
273 if input.size % 8:
274 output.copyBitsFrom(input.stream,
275 input.absolute_address, input.size, input.endian)
276 else:
277 output.copyBytesFrom(input.stream,
278 input.absolute_address, input.size//8)
279 else:
280 # Altered: call writeInto() method of each field
281 realaddr = 0
282 for field in self:
283 field.writeInto(output)
284 realaddr += field.size
285
286 def _getValue(self):
287 raise EditorError('Field set "%s" has no value' % self.path)
288 def _setValue(self, value):
289 raise EditorError('Field set "%s" value is read only' % self.path)
290 value = property(_getValue, _setValue, "Value of field")
291
292class EditableFloat(EditableFieldSet):
293 _value = None
294
295 def _isAltered(self):
296 return (self._value is not None)
297 is_altered = property(_isAltered)
298
299 def writeInto(self, output):
300 if self._value is not None:
301 self._write(output)
302 else:
303 EditableFieldSet.writeInto(self, output)
304
305 def _write(self, output):
306 format = self.input.struct_format
307 raw = struct.pack(format, self._value)
308 output.writeBytes(raw)
309
310 def _setValue(self, value):
311 self.parent._is_altered = True
312 self._value = value
313 value = property(EditableFieldSet._getValue, _setValue)
314
315def createEditableFieldSet(parent, field):
316 cls = field.__class__
317 # FIXME: Support Float80
318 if cls in (Float32, Float64):
319 return EditableFloat(parent, field)
320 else:
321 return EditableFieldSet(parent, field)
322
323class NewFieldSet(EditableFieldSet):
324 def __init__(self, parent, name):
325 EditableFieldSet.__init__(self, parent, None)
326 self._name = name
327 self._endian = parent.endian
328
329 def __iter__(self):
330 if None in self._inserted:
331 return iter(self._inserted[None])
332 else:
333 raise StopIteration()
334
335 def _getName(self):
336 return self._name
337 name = property(_getName)
338
339 def _getEndian(self):
340 return self._endian
341 endian = property(_getEndian)
342
343 is_altered = property(lambda self: True)
344
345def createEditor(fieldset):
346 return EditableFieldSet(None, fieldset)
347
diff --git a/lib/hachoir_editor/typed_field.py b/lib/hachoir_editor/typed_field.py
deleted file mode 100644
index 0f0427b..0000000
--- a/lib/hachoir_editor/typed_field.py
+++ /dev/null
@@ -1,253 +0,0 @@
1from hachoir_core.field import (
2 RawBits, Bit, Bits, PaddingBits,
3 RawBytes, Bytes, PaddingBytes,
4 GenericString, Character,
5 isInteger, isString)
6from field import FakeField
7
8class EditableField(FakeField):
9 """
10 Pure virtual class used to write editable field class.
11 """
12
13 _is_altered = False
14 def __init__(self, parent, name, value=None):
15 FakeField.__init__(self, parent, name)
16 self._value = value
17
18 def _isAltered(self):
19 return self._is_altered
20 is_altered = property(_isAltered)
21
22 def hasValue(self):
23 return True
24
25 def _computeSize(self):
26 raise NotImplementedError()
27 def _getValue(self):
28 return self._value
29 def _setValue(self, value):
30 self._value = value
31
32 def _propGetValue(self):
33 if self._value is not None:
34 return self._getValue()
35 else:
36 return FakeField._getValue(self)
37 def _propSetValue(self, value):
38 self._setValue(value)
39 self._is_altered = True
40 value = property(_propGetValue, _propSetValue)
41
42 def _getSize(self):
43 if self._value is not None:
44 return self._computeSize()
45 else:
46 return FakeField._getSize(self)
47 size = property(_getSize)
48
49 def _write(self, output):
50 raise NotImplementedError()
51
52 def writeInto(self, output):
53 if self._is_altered:
54 self._write(output)
55 else:
56 return FakeField.writeInto(self, output)
57
58class EditableFixedField(EditableField):
59 """
60 Editable field with fixed size.
61 """
62
63 def __init__(self, parent, name, value=None, size=None):
64 EditableField.__init__(self, parent, name, value)
65 if size is not None:
66 self._size = size
67 else:
68 self._size = self._parent._getOriginalField(self._name).size
69
70 def _getSize(self):
71 return self._size
72 size = property(_getSize)
73
74class EditableBits(EditableFixedField):
75 def __init__(self, parent, name, *args):
76 if args:
77 if len(args) != 2:
78 raise TypeError(
79 "Wrong argument count, EditableBits constructor prototype is: "
80 "(parent, name, [size, value])")
81 size = args[0]
82 value = args[1]
83 assert isinstance(value, (int, long))
84 else:
85 size = None
86 value = None
87 EditableFixedField.__init__(self, parent, name, value, size)
88 if args:
89 self._setValue(args[1])
90 self._is_altered = True
91
92 def _setValue(self, value):
93 if not(0 <= value < (1 << self._size)):
94 raise ValueError("Invalid value, must be in range %s..%s"
95 % (0, (1 << self._size) - 1))
96 self._value = value
97
98 def _write(self, output):
99 output.writeBits(self._size, self._value, self._parent.endian)
100
101class EditableBytes(EditableField):
102 def _setValue(self, value):
103 if not value: raise ValueError(
104 "Unable to set empty string to a EditableBytes field")
105 self._value = value
106
107 def _computeSize(self):
108 return len(self._value) * 8
109
110 def _write(self, output):
111 output.writeBytes(self._value)
112
113class EditableString(EditableField):
114 MAX_SIZE = {
115 "Pascal8": (1 << 8)-1,
116 "Pascal16": (1 << 16)-1,
117 "Pascal32": (1 << 32)-1,
118 }
119
120 def __init__(self, parent, name, *args, **kw):
121 if len(args) == 2:
122 value = args[1]
123 assert isinstance(value, str) # TODO: support Unicode
124 elif not args:
125 value = None
126 else:
127 raise TypeError(
128 "Wrong argument count, EditableString constructor prototype is:"
129 "(parent, name, [format, value])")
130 EditableField.__init__(self, parent, name, value)
131 if len(args) == 2:
132 self._charset = kw.get('charset', None)
133 self._format = args[0]
134 if self._format in GenericString.PASCAL_FORMATS:
135 self._prefix_size = GenericString.PASCAL_FORMATS[self._format]
136 else:
137 self._prefix_size = 0
138 self._suffix_str = GenericString.staticSuffixStr(
139 self._format, self._charset, self._parent.endian)
140 self._is_altered = True
141 else:
142 orig = self._parent._getOriginalField(name)
143 self._charset = orig.charset
144 self._format = orig.format
145 self._prefix_size = orig.content_offset
146 self._suffix_str = orig.suffix_str
147
148 def _setValue(self, value):
149 size = len(value)
150 if self._format in self.MAX_SIZE and self.MAX_SIZE[self._format] < size:
151 raise ValueError("String is too big")
152 self._value = value
153
154 def _computeSize(self):
155 return (self._prefix_size + len(self._value) + len(self._suffix_str))*8
156
157 def _write(self, output):
158 if self._format in GenericString.SUFFIX_FORMAT:
159 output.writeBytes(self._value)
160 output.writeBytes(self._suffix_str)
161 elif self._format == "fixed":
162 output.writeBytes(self._value)
163 else:
164 assert self._format in GenericString.PASCAL_FORMATS
165 size = GenericString.PASCAL_FORMATS[self._format]
166 output.writeInteger(len(self._value), False, size, self._parent.endian)
167 output.writeBytes(self._value)
168
169class EditableCharacter(EditableFixedField):
170 def __init__(self, parent, name, *args):
171 if args:
172 if len(args) != 3:
173 raise TypeError(
174 "Wrong argument count, EditableCharacter "
175 "constructor prototype is: (parent, name, [value])")
176 value = args[0]
177 if not isinstance(value, str) or len(value) != 1:
178 raise TypeError("EditableCharacter needs a character")
179 else:
180 value = None
181 EditableFixedField.__init__(self, parent, name, value, 8)
182 if args:
183 self._is_altered = True
184
185 def _setValue(self, value):
186 if not isinstance(value, str) or len(value) != 1:
187 raise TypeError("EditableCharacter needs a character")
188 self._value = value
189
190 def _write(self, output):
191 output.writeBytes(self._value)
192
193class EditableInteger(EditableFixedField):
194 VALID_VALUE_SIGNED = {
195 8: (-(1 << 8), (1 << 8)-1),
196 16: (-(1 << 15), (1 << 15)-1),
197 32: (-(1 << 31), (1 << 31)-1),
198 }
199 VALID_VALUE_UNSIGNED = {
200 8: (0, (1 << 8)-1),
201 16: (0, (1 << 16)-1),
202 32: (0, (1 << 32)-1)
203 }
204
205 def __init__(self, parent, name, *args):
206 if args:
207 if len(args) != 3:
208 raise TypeError(
209 "Wrong argument count, EditableInteger constructor prototype is: "
210 "(parent, name, [signed, size, value])")
211 size = args[1]
212 value = args[2]
213 assert isinstance(value, (int, long))
214 else:
215 size = None
216 value = None
217 EditableFixedField.__init__(self, parent, name, value, size)
218 if args:
219 self._signed = args[0]
220 self._is_altered = True
221 else:
222 self._signed = self._parent._getOriginalField(self._name).signed
223
224 def _setValue(self, value):
225 if self._signed:
226 valid = self.VALID_VALUE_SIGNED
227 else:
228 valid = self.VALID_VALUE_UNSIGNED
229 minval, maxval = valid[self._size]
230 if not(minval <= value <= maxval):
231 raise ValueError("Invalid value, must be in range %s..%s"
232 % (minval, maxval))
233 self._value = value
234
235 def _write(self, output):
236 output.writeInteger(
237 self.value, self._signed, self._size//8, self._parent.endian)
238
239def createEditableField(fieldset, field):
240 if isInteger(field):
241 cls = EditableInteger
242 elif isString(field):
243 cls = EditableString
244 elif field.__class__ in (RawBytes, Bytes, PaddingBytes):
245 cls = EditableBytes
246 elif field.__class__ in (RawBits, Bits, Bit, PaddingBits):
247 cls = EditableBits
248 elif field.__class__ == Character:
249 cls = EditableCharacter
250 else:
251 cls = FakeField
252 return cls(fieldset, field.name)
253
diff --git a/lib/images.py b/lib/images.py
deleted file mode 100644
index d090015..0000000
--- a/lib/images.py
+++ /dev/null
@@ -1,37 +0,0 @@
1'''
2 Takes care about pictures formats
3'''
4
5import parser
6
7
8class JpegStripper(parser.GenericParser):
9 '''
10 represents a jpeg file
11 '''
12 def _should_remove(self, field):
13 '''
14 return True if the field is compromizing
15 '''
16 if field.name.startswith('comment'):
17 return True
18 elif field.name in ("photoshop", "exif", "adobe"):
19 return True
20 else:
21 return False
22
23
24class PngStripper(parser.GenericParser):
25 '''
26 represents a png file
27 '''
28 def _should_remove(self, field):
29 '''
30 return True if the field is compromizing
31 '''
32 if field.name.startswith("text["):
33 return True
34 elif field.name is "time":
35 return True
36 else:
37 return False
diff --git a/lib/mat.py b/lib/mat.py
deleted file mode 100644
index fd13287..0000000
--- a/lib/mat.py
+++ /dev/null
@@ -1,162 +0,0 @@
1#!/usr/bin/env python
2
3'''
4 Metadata anonymisation toolkit library
5'''
6
7import os
8import subprocess
9import logging
10import mimetypes
11import xml.sax
12
13import hachoir_core.cmd_line
14import hachoir_parser
15
16import images
17import audio
18import office
19import archive
20import misc
21
22__version__ = '0.1'
23__author__ = 'jvoisin'
24
25LOGGING_LEVEL = logging.DEBUG
26
27logging.basicConfig(level=LOGGING_LEVEL)
28
29STRIPPERS = {
30 'application/x-tar': archive.TarStripper,
31 'application/x-gzip': archive.GzipStripper,
32 'application/x-bzip2': archive.Bzip2Stripper,
33 'application/zip': archive.ZipStripper,
34 'audio/mpeg': audio.MpegAudioStripper,
35 'image/jpeg': images.JpegStripper,
36 'image/png': images.PngStripper,
37 'application/x-bittorrent': misc.TorrentStripper,
38 'application/opendocument': office.OpenDocumentStripper,
39 'application/officeopenxml': office.OpenXmlStripper,
40}
41
42try:
43 import poppler
44 import cairo
45 STRIPPERS['application/x-pdf'] = office.PdfStripper
46 STRIPPERS['application/pdf'] = office.PdfStripper
47except ImportError:
48 print('Unable to import python-poppler and/or python-cairo: no pdf \
49 support')
50
51try:
52 import mutagen
53 STRIPPERS['audio/x-flac'] = audio.FlacStripper
54 STRIPPERS['audio/vorbis'] = audio.OggStripper
55except ImportError:
56 print('unable to import python-mutagen : limited audio format support')
57
58
59class XMLParser(xml.sax.handler.ContentHandler):
60 '''
61 Parse the supported format xml, and return a corresponding
62 list of dict
63 '''
64 def __init__(self):
65 self.dict = {}
66 self.list = []
67 self.content, self.key = '', ''
68 self.between = False
69
70 def startElement(self, name, attrs):
71 '''
72 Called when entering into xml balise
73 '''
74 self.between = True
75 self.key = name
76 self.content = ''
77
78 def endElement(self, name):
79 '''
80 Called when exiting a xml balise
81 '''
82 if name == 'format': # exiting a fileformat section
83 self.list.append(self.dict.copy())
84 self.dict.clear()
85 else:
86 content = self.content.replace('\s', ' ')
87 self.dict[self.key] = content
88 self.between = False
89
90 def characters(self, characters):
91 '''
92 Concatenate the content between opening and closing balises
93 '''
94 if self.between is True:
95 self.content += characters
96
97
98def secure_remove(filename):
99 '''
100 securely remove the file
101 '''
102 removed = False
103 try:
104 subprocess.call('shred --remove %s' % filename, shell=True)
105 removed = True
106 except:
107 logging.error('Unable to securely remove %s' % filename)
108
109 if removed is False:
110 try:
111 os.remove(filename)
112 except:
113 logging.error('Unable to remove %s' % filename)
114
115
116def is_secure(filename):
117 '''
118 Prevent shell injection
119 '''
120 if not(os.path.isfile(filename)): # check if the file exist
121 logging.error('%s is not a valid file' % filename)
122 return False
123 else:
124 return True
125
126
127def create_class_file(name, backup, add2archive):
128 '''
129 return a $FILETYPEStripper() class,
130 corresponding to the filetype of the given file
131 '''
132 if not is_secure(name):
133 return
134
135 filename = ''
136 try:
137 filename = hachoir_core.cmd_line.unicodeFilename(name)
138 except TypeError: # get rid of "decoding Unicode is not supported"
139 filename = name
140
141 parser = hachoir_parser.createParser(filename)
142 if not parser:
143 logging.info('Unable to parse %s' % filename)
144 return
145
146 mime = parser.mime_type
147
148 if mime == 'application/zip': # some formats are zipped stuff
149 mime = mimetypes.guess_type(name)[0]
150
151 if mime.startswith('application/vnd.oasis.opendocument'):
152 mime = 'application/opendocument' # opendocument fileformat
153 elif mime.startswith('application/vnd.openxmlformats-officedocument'):
154 mime = 'application/officeopenxml' # office openxml
155
156 try:
157 stripper_class = STRIPPERS[mime]
158 except KeyError:
159 logging.info('Don\'t have stripper for %s format' % mime)
160 return
161
162 return stripper_class(filename, parser, mime, backup, add2archive)
diff --git a/lib/misc.py b/lib/misc.py
deleted file mode 100644
index fdf53b2..0000000
--- a/lib/misc.py
+++ /dev/null
@@ -1,63 +0,0 @@
1'''
2 Care about misc formats
3'''
4
5import parser
6
7from bencode import bencode
8#import bencode
9
10
11class TorrentStripper(parser.GenericParser):
12 '''
13 Represent a torrent file with the help
14 of the bencode lib from Petru Paler
15 '''
16 def __init__(self, filename, parser, mime, backup, add2archive):
17 super(TorrentStripper, self).__init__(filename, parser, mime,
18 backup, add2archive)
19 self.fields = ['comment', 'creation date', 'created by']
20
21 def is_clean(self):
22 '''
23 Check if the file is clean from harmful metadatas
24 '''
25 with open(self.filename, 'r') as f:
26 decoded = bencode.bdecode(f.read())
27 for key in self.fields:
28 try:
29 if decoded[key] != '':
30 return False
31 except:
32 pass
33 return True
34
35 def get_meta(self):
36 '''
37 Return a dict with all the meta of the file
38 '''
39 metadata = {}
40 with open(self.filename, 'r') as f:
41 decoded = bencode.bdecode(f.read())
42 for key in self.fields:
43 try:
44 if decoded[key] != '':
45 metadata[key] = decoded[key]
46 except:
47 pass
48 return metadata
49
50 def remove_all(self):
51 '''
52 Remove all the files that are compromizing
53 '''
54 with open(self.filename, 'r') as f:
55 decoded = bencode.bdecode(f.read())
56 for key in self.fields:
57 try:
58 decoded[key] = ''
59 except:
60 pass
61 with open(self.output, 'w') as f: # encode the decoded torrent
62 f.write(bencode.bencode(decoded)) # and write it in self.output
63 self.do_backup()
diff --git a/lib/office.py b/lib/office.py
deleted file mode 100644
index 33af48e..0000000
--- a/lib/office.py
+++ /dev/null
@@ -1,257 +0,0 @@
1'''
2 Care about office's formats
3'''
4
5import os
6import logging
7import zipfile
8import fileinput
9
10try:
11 import cairo
12 import poppler
13except ImportError:
14 pass
15
16import mat
17import parser
18import archive
19import pdfrw
20
21
22class OpenDocumentStripper(archive.GenericArchiveStripper):
23 '''
24 An open document file is a zip, with xml file into.
25 The one that interest us is meta.xml
26 '''
27
28 def get_meta(self):
29 '''
30 Return a dict with all the meta of the file by
31 trying to read the meta.xml file.
32 '''
33 zipin = zipfile.ZipFile(self.filename, 'r')
34 metadata = {}
35 try:
36 content = zipin.read('meta.xml')
37 zipin.close()
38 metadata[self.filename] = 'harful meta'
39 except KeyError: # no meta.xml file found
40 logging.debug('%s has no opendocument metadata' % self.filename)
41 return metadata
42
43 def _remove_all(self, method):
44 '''
45 FIXME ?
46 There is a patch implementing the Zipfile.remove()
47 method here : http://bugs.python.org/issue6818
48 '''
49 zipin = zipfile.ZipFile(self.filename, 'r')
50 zipout = zipfile.ZipFile(self.output, 'w', allowZip64=True)
51
52 for item in zipin.namelist():
53 name = os.path.join(self.tempdir, item)
54 _, ext = os.path.splitext(name)
55
56 if item.endswith('manifest.xml'):
57 # contain the list of all files present in the archive
58 zipin.extract(item, self.tempdir)
59 for line in fileinput.input(name, inplace=1):
60 #remove the line which contains "meta.xml"
61 line = line.strip()
62 if not 'meta.xml' in line:
63 print line
64 zipout.write(name, item)
65
66 elif ext in parser.NOMETA or item == 'mimetype':
67 #keep NOMETA files, and the "manifest" file
68 if item != 'meta.xml': # contains the metadata
69 zipin.extract(item, self.tempdir)
70 zipout.write(name, item)
71
72 else:
73 zipin.extract(item, self.tempdir)
74 if os.path.isfile(name):
75 try:
76 cfile = mat.create_class_file(name, False,
77 self.add2archive)
78 if method == 'normal':
79 cfile.remove_all()
80 else:
81 cfile.remove_all_ugly()
82 logging.debug('Processing %s from %s' % (item,
83 self.filename))
84 zipout.write(name, item)
85 except:
86 logging.info('%s\' fileformat is not supported' % item)
87 if self.add2archive:
88 zipout.write(name, item)
89 zipout.comment = ''
90 logging.info('%s treated' % self.filename)
91 zipin.close()
92 zipout.close()
93 self.do_backup()
94
95 def is_clean(self):
96 '''
97 Check if the file is clean from harmful metadatas
98 '''
99 zipin = zipfile.ZipFile(self.filename, 'r')
100 try:
101 zipin.getinfo('meta.xml')
102 except KeyError: # no meta.xml in the file
103 czf = archive.ZipStripper(self.filename, self.parser,
104 'application/zip', self.backup, self.add2archive)
105 if czf.is_clean():
106 zipin.close()
107 return True
108 zipin.close()
109 return False
110
111
112class PdfStripper(parser.GenericParser):
113 '''
114 Represent a pdf file
115 '''
116 def __init__(self, filename, parser, mime, backup, add2archive):
117 super(PdfStripper, self).__init__(filename, parser, mime, backup,
118 add2archive)
119 uri = 'file://' + os.path.abspath(self.filename)
120 self.password = None
121 self.document = poppler.document_new_from_file(uri, self.password)
122 self.meta_list = ('title', 'author', 'subject', 'keywords', 'creator',
123 'producer', 'creation-date', 'mod-date', 'metadata')
124
125 def is_clean(self):
126 '''
127 Check if the file is clean from harmful metadatas
128 '''
129 for key in self.meta_list:
130 if key == 'creation-date' or key == 'mod-date':
131 if self.document.get_property(key) != -1:
132 return False
133 elif self.document.get_property(key) is not None and \
134 self.document.get_property(key) != '':
135 return False
136 return True
137
138 def remove_all(self):
139 '''
140 Opening the pdf with poppler, then doing a render
141 on a cairo pdfsurface for each pages.
142 Thanks to Lunar^for the idea.
143 http://cairographics.org/documentation/pycairo/2/
144 python-poppler is not documented at all : have fun ;)
145 '''
146 page = self.document.get_page(0)
147 page_width, page_height = page.get_size()
148 surface = cairo.PDFSurface(self.output, page_width, page_height)
149 context = cairo.Context(surface) # context draws on the surface
150 logging.debug('Pdf rendering of %s' % self.filename)
151 for pagenum in xrange(self.document.get_n_pages()):
152 page = self.document.get_page(pagenum)
153 context.translate(0, 0)
154 page.render(context) # render the page on context
155 context.show_page() # draw context on surface
156 surface.finish()
157
158 #For now, poppler cannot write meta, so we must use pdfrw
159 logging.debug('Removing %s\'s superficial metadata' % self.filename)
160 trailer = pdfrw.PdfReader(self.output)
161 trailer.Info.Producer = trailer.Info.Creator = None
162 writer = pdfrw.PdfWriter()
163 writer.trailer = trailer
164 writer.write(self.output)
165 self.do_backup()
166
167 def get_meta(self):
168 '''
169 Return a dict with all the meta of the file
170 '''
171 metadata = {}
172 for key in self.meta_list:
173 if key == 'creation-date' or key == 'mod-date':
174 #creation and modification are set to -1
175 if self.document.get_property(key) != -1:
176 metadata[key] = self.document.get_property(key)
177 elif self.document.get_property(key) is not None and \
178 self.document.get_property(key) != '':
179 metadata[key] = self.document.get_property(key)
180 return metadata
181
182
183class OpenXmlStripper(archive.GenericArchiveStripper):
184 '''
185 Represent an office openxml document, which is like
186 an opendocument format, with some tricky stuff added.
187 It contains mostly xml, but can have media blobs, crap, ...
188 (I don't like this format.)
189 '''
190 def _remove_all(self, method):
191 '''
192 FIXME ?
193 There is a patch implementing the Zipfile.remove()
194 method here : http://bugs.python.org/issue6818
195 '''
196 zipin = zipfile.ZipFile(self.filename, 'r')
197 zipout = zipfile.ZipFile(self.output, 'w',
198 allowZip64=True)
199 for item in zipin.namelist():
200 name = os.path.join(self.tempdir, item)
201 _, ext = os.path.splitext(name)
202 if item.startswith('docProps/'): # metadatas
203 pass
204 elif ext in parser.NOMETA or item == '.rels':
205 #keep parser.NOMETA files, and the file named ".rels"
206 zipin.extract(item, self.tempdir)
207 zipout.write(name, item)
208 else:
209 zipin.extract(item, self.tempdir)
210 if os.path.isfile(name): # don't care about folders
211 try:
212 cfile = mat.create_class_file(name, False,
213 self.add2archive)
214 if method == 'normal':
215 cfile.remove_all()
216 else:
217 cfile.remove_all_ugly()
218 logging.debug('Processing %s from %s' % (item,
219 self.filename))
220 zipout.write(name, item)
221 except:
222 logging.info('%s\' fileformat is not supported' % item)
223 if self.add2archive:
224 zipout.write(name, item)
225 zipout.comment = ''
226 logging.info('%s treated' % self.filename)
227 zipin.close()
228 zipout.close()
229 self.do_backup()
230
231 def is_clean(self):
232 '''
233 Check if the file is clean from harmful metadatas
234 '''
235 zipin = zipfile.ZipFile(self.filename, 'r')
236 for item in zipin.namelist():
237 if item.startswith('docProps/'):
238 return False
239 zipin.close()
240 czf = archive.ZipStripper(self.filename, self.parser,
241 'application/zip', self.backup, self.add2archive)
242 if not czf.is_clean():
243 return False
244 else:
245 return True
246
247 def get_meta(self):
248 '''
249 Return a dict with all the meta of the file
250 '''
251 zipin = zipfile.ZipFile(self.filename, 'r')
252 metadata = {}
253 for item in zipin.namelist():
254 if item.startswith('docProps/'):
255 metadata[item] = 'harmful content'
256 zipin.close()
257 return metadata
diff --git a/lib/parser.py b/lib/parser.py
deleted file mode 100644
index 58dd7fa..0000000
--- a/lib/parser.py
+++ /dev/null
@@ -1,104 +0,0 @@
1'''
2 Parent class of all parser
3'''
4
5import hachoir_core
6import hachoir_editor
7
8import os
9
10import mat
11
12NOMETA = ('.bmp', '.rdf', '.txt', '.xml', '.rels')
13#bmp : image
14#rdf : text
15#txt : plain text
16#xml : formated text
17#rels : openxml foramted text
18
19
20class GenericParser(object):
21 '''
22 Parent class of all parsers
23 '''
24 def __init__(self, filename, parser, mime, backup, add2archive):
25 self.filename = ''
26 self.parser = parser
27 self.mime = mime
28 self.backup = backup
29 self.editor = hachoir_editor.createEditor(parser)
30 self.realname = filename
31 try:
32 self.filename = hachoir_core.cmd_line.unicodeFilename(filename)
33 except TypeError: # get rid of "decoding Unicode is not supported"
34 self.filename = filename
35 basename, ext = os.path.splitext(filename)
36 self.output = basename + '.cleaned' + ext
37 self.basename = os.path.basename(filename) # only filename
38
39 def is_clean(self):
40 '''
41 Check if the file is clean from harmful metadatas
42 '''
43 for field in self.editor:
44 if self._should_remove(field):
45 return False
46 return True
47
48 def remove_all(self):
49 '''
50 Remove all the files that are compromizing
51 '''
52 for field in self.editor:
53 if self._should_remove(field):
54 self._remove(field.name)
55 hachoir_core.field.writeIntoFile(self.editor, self.output)
56 self.do_backup()
57
58 def remove_all_ugly(self):
59 '''
60 If the remove_all() is not efficient enough,
61 this method is implemented :
62 It is efficient, but destructive.
63 In a perfect world, with nice fileformat,
64 this method would not exist.
65 '''
66 self.remove_all()
67
68 def _remove(self, field):
69 '''
70 Delete the given field
71 '''
72 del self.editor[field]
73
74 def get_meta(self):
75 '''
76 Return a dict with all the meta of the file
77 '''
78 metadata = {}
79 for field in self.editor:
80 if self._should_remove(field):
81 try:
82 metadata[field.name] = field.value
83 except:
84 metadata[field.name] = 'harmful content'
85 return metadata
86
87 def _should_remove(self, key):
88 '''
89 return True if the field is compromizing
90 abstract method
91 '''
92 raise NotImplementedError
93
94 def do_backup(self):
95 '''
96 Do a backup of the file if asked,
97 and change his creation/access date
98 '''
99 if self.backup is True:
100 os.utime(self.output, (0, 0))
101 else:
102 mat.secure_remove(self.filename)
103 os.rename(self.output, self.filename)
104 os.utime(self.filename, (0, 0))
diff --git a/lib/pdfrw/__init__.py b/lib/pdfrw/__init__.py
deleted file mode 100644
index 26e8c73..0000000
--- a/lib/pdfrw/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
1# A part of pdfrw (pdfrw.googlecode.com)
2# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas
3# MIT license -- See LICENSE.txt for details
4
5from pdfwriter import PdfWriter
6from pdfreader import PdfReader
7from pdfobjects import PdfObject, PdfName, PdfArray, PdfDict, IndirectPdfDict, PdfString
8from pdftokens import PdfTokens
9
10# Add a tiny bit of compatibility to pyPdf
11
12PdfFileReader = PdfReader
13PdfFileWriter = PdfWriter
14
diff --git a/lib/pdfrw/pdfcompress.py b/lib/pdfrw/pdfcompress.py
deleted file mode 100644
index 1c11970..0000000
--- a/lib/pdfrw/pdfcompress.py
+++ /dev/null
@@ -1,57 +0,0 @@
1# A part of pdfrw (pdfrw.googlecode.com)
2# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas
3# MIT license -- See LICENSE.txt for details
4
5'''
6Currently, this sad little file only knows how to decompress
7using the flate (zlib) algorithm. Maybe more later, but it's
8not a priority for me...
9'''
10
11from __future__ import generators
12
13try:
14 set
15except NameError:
16 from sets import Set as set
17
18import zlib
19from pdfobjects import PdfDict, PdfName
20
21
22def streamobjects(mylist):
23 for obj in mylist:
24 if isinstance(obj, PdfDict) and obj.stream is not None:
25 yield obj
26
27def uncompress(mylist, warnings=set()):
28 flate = PdfName.FlateDecode
29 for obj in streamobjects(mylist):
30 ftype = obj.Filter
31 if ftype is None:
32 continue
33 if isinstance(ftype, list) and len(ftype) == 1:
34 # todo: multiple filters
35 ftype = ftype[0]
36 parms = obj.DecodeParms
37 if ftype != flate or parms is not None:
38 msg = 'Not decompressing: cannot use filter %s with parameters %s' % (repr(ftype), repr(parms))
39 if msg not in warnings:
40 warnings.add(msg)
41 print msg
42 else:
43 obj.stream = zlib.decompress(obj.stream)
44 obj.Filter = None
45
46def compress(mylist):
47 flate = PdfName.FlateDecode
48 for obj in streamobjects(mylist):
49 ftype = obj.Filter
50 if ftype is not None:
51 continue
52 oldstr = obj.stream
53 newstr = zlib.compress(oldstr)
54 if len(newstr) < len(oldstr) + 30:
55 obj.stream = newstr
56 obj.Filter = flate
57 obj.DecodeParms = None
diff --git a/lib/pdfrw/pdfobjects.py b/lib/pdfrw/pdfobjects.py
deleted file mode 100644
index 08ad825..0000000
--- a/lib/pdfrw/pdfobjects.py
+++ /dev/null
@@ -1,183 +0,0 @@
1# A part of pdfrw (pdfrw.googlecode.com)
2# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas
3# MIT license -- See LICENSE.txt for details
4
5'''
6Objects that can occur in PDF files. The most important
7objects are arrays and dicts. Either of these can be
8indirect or not, and dicts could have an associated
9stream.
10'''
11from __future__ import generators
12
13try:
14 set
15except NameError:
16 from sets import Set as set
17
18import re
19
20class PdfObject(str):
21 indirect = False
22
23class PdfArray(list):
24 indirect = False
25
26class PdfName(object):
27 def __getattr__(self, name):
28 return self(name)
29 def __call__(self, name):
30 return PdfObject('/' + name)
31
32PdfName = PdfName()
33
34class PdfString(str):
35 indirect = False
36 unescape_dict = {'\\b':'\b', '\\f':'\f', '\\n':'\n',
37 '\\r':'\r', '\\t':'\t',
38 '\\\r\n': '', '\\\r':'', '\\\n':'',
39 '\\\\':'\\', '\\':'',
40 }
41 unescape_pattern = r'(\\b|\\f|\\n|\\r|\\t|\\\r\n|\\\r|\\\n|\\[0-9]+|\\)'
42 unescape_func = re.compile(unescape_pattern).split
43
44 hex_pattern = '([a-fA-F0-9][a-fA-F0-9]|[a-fA-F0-9])'
45 hex_func = re.compile(hex_pattern).split
46
47 hex_pattern2 = '([a-fA-F0-9][a-fA-F0-9][a-fA-F0-9][a-fA-F0-9]|[a-fA-F0-9][a-fA-F0-9]|[a-fA-F0-9])'
48 hex_func2 = re.compile(hex_pattern2).split
49
50 hex_funcs = hex_func, hex_func2
51
52 indirect = False
53
54 def decode_regular(self, remap=chr):
55 assert self[0] == '(' and self[-1] == ')'
56 mylist = self.unescape_func(self[1:-1])
57 result = []
58 unescape = self.unescape_dict.get
59 for chunk in mylist:
60 chunk = unescape(chunk, chunk)
61 if chunk.startswith('\\') and len(chunk) > 1:
62 value = int(chunk[1:], 8)
63 # FIXME: TODO: Handle unicode here
64 if value > 127:
65 value = 127
66 chunk = remap(value)
67 if chunk:
68 result.append(chunk)
69 return ''.join(result)
70
71 def decode_hex(self, remap=chr, twobytes=False):
72 data = ''.join(self.split())
73 data = self.hex_funcs[twobytes](data)
74 chars = data[1::2]
75 other = data[0::2]
76 assert other[0] == '<' and other[-1] == '>' and ''.join(other) == '<>', self
77 return ''.join([remap(int(x, 16)) for x in chars])
78
79 def decode(self, remap=chr, twobytes=False):
80 if self.startswith('('):
81 return self.decode_regular(remap)
82
83 else:
84 return self.decode_hex(remap, twobytes)
85
86 def encode(cls, source, usehex=False):
87 assert not usehex, "Not supported yet"
88 if isinstance(source, unicode):
89 source = source.encode('utf-8')
90 else:
91 source = str(source)
92 source = source.replace('\\', '\\\\')
93 source = source.replace('(', '\\(')
94 source = source.replace(')', '\\)')
95 return cls('(' +source + ')')
96 encode = classmethod(encode)
97
98class PdfDict(dict):
99 indirect = False
100 stream = None
101
102 _special = dict(indirect = ('indirect', False),
103 stream = ('stream', True),
104 _stream = ('stream', False),
105 )
106
107 def __setitem__(self, name, value):
108 assert name.startswith('/'), name
109 if value is not None:
110 dict.__setitem__(self, name, value)
111 elif name in self:
112 del self[name]
113
114 def __init__(self, *args, **kw):
115 if args:
116 if len(args) == 1:
117 args = args[0]
118 self.update(args)
119 if isinstance(args, PdfDict):
120 self.indirect = args.indirect
121 self._stream = args.stream
122 for key, value in kw.iteritems():
123 setattr(self, key, value)
124
125 def __getattr__(self, name):
126 return self.get(PdfName(name))
127
128 def __setattr__(self, name, value):
129 info = self._special.get(name)
130 if info is None:
131 self[PdfName(name)] = value
132 else:
133 name, setlen = info
134 self.__dict__[name] = value
135 if setlen:
136 notnone = value is not None
137 self.Length = notnone and PdfObject(len(value)) or None
138
139 def iteritems(self):
140 for key, value in dict.iteritems(self):
141 if value is not None:
142 assert key.startswith('/'), (key, value)
143 yield key, value
144
145 def inheritable(self):
146 ''' Search through ancestors as needed for inheritable
147 dictionary items
148 '''
149 class Search(object):
150 def __init__(self, basedict):
151 self.basedict = basedict
152 def __getattr__(self, name):
153 return self[name]
154 def __getitem__(self, name):
155 visited = set()
156 mydict = self.basedict
157 while 1:
158 value = getattr(mydict, name)
159 if value is not None:
160 return value
161 myid = id(mydict)
162 assert myid not in visited
163 visited.add(myid)
164 mydict = mydict.Parent
165 if mydict is None:
166 return
167 return Search(self)
168 inheritable = property(inheritable)
169
170 def private(self):
171 ''' Allows setting private metadata for use in
172 processing (not sent to PDF file)
173 '''
174 class Private(object):
175 pass
176
177 result = Private()
178 result.__dict__ = self.__dict__
179 return result
180 private = property(private)
181
182class IndirectPdfDict(PdfDict):
183 indirect = True
diff --git a/lib/pdfrw/pdfreader.py b/lib/pdfrw/pdfreader.py
deleted file mode 100644
index 6f57bea..0000000
--- a/lib/pdfrw/pdfreader.py
+++ /dev/null
@@ -1,213 +0,0 @@
1# A part of pdfrw (pdfrw.googlecode.com)
2# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas
3# MIT license -- See LICENSE.txt for details
4
5'''
6The PdfReader class reads an entire PDF file into memory and
7parses the top-level container objects. (It does not parse
8into streams.) The object subclasses PdfDict, and the
9document pages are stored in a list in the pages attribute
10of the object.
11'''
12
13from pdftokens import PdfTokens
14from pdfobjects import PdfDict, PdfArray, PdfName
15from pdfcompress import uncompress
16
17class PdfReader(PdfDict):
18
19 class unresolved:
20 # Used as a placeholder until we have an object.
21 pass
22
23 def readindirect(self, objnum, gennum):
24 ''' Read an indirect object. If it has already
25 been read, return it from the cache.
26 '''
27
28 def setobj(obj):
29 # Store the new object in the dictionary
30 # once we have its value
31 record[1] = obj
32
33 def ordinary(source, setobj, obj):
34 # Deal with an ordinary (non-array, non-dict) object
35 setobj(obj)
36 return obj
37
38 fdata, objnum, gennum = self.fdata, int(objnum), int(gennum)
39 record = self.indirect_objects[fdata, objnum, gennum]
40 if record[1] is not self.unresolved:
41 return record[1]
42
43 # Read the object header and validate it
44 source = PdfTokens(fdata, record[0])
45 objid = source.multiple(3)
46 assert int(objid[0]) == objnum, objid
47 assert int(objid[1]) == gennum, objid
48 assert objid[2] == 'obj', objid
49
50 # Read the object, and call special code if it starts
51 # an array or dictionary
52 obj = source.next()
53 obj = self.special.get(obj, ordinary)(source, setobj, obj)
54 self.readstream(obj, source)
55 obj.indirect = True
56 return obj
57
58 def readstream(obj, source):
59 ''' Read optional stream following a dictionary
60 object.
61 '''
62 tok = source.next()
63 if tok == 'endobj':
64 return # No stream
65
66 assert isinstance(obj, PdfDict)
67 assert tok == 'stream', tok
68 fdata = source.fdata
69 floc = fdata.rindex(tok, 0, source.floc) + len(tok)
70 ch = fdata[floc]
71 if ch == '\r':
72 floc += 1
73 ch = fdata[floc]
74 assert ch == '\n'
75 startstream = floc + 1
76 endstream = startstream + int(obj.Length)
77 obj._stream = fdata[startstream:endstream]
78 source = PdfTokens(fdata, endstream)
79 endit = source.multiple(2)
80 if endit != 'endstream endobj'.split():
81 # /Length attribute is broken, try to read stream
82 # anyway disregarding the specified value
83 # TODO: issue warning here once we have some kind of
84 # logging
85 endstream = fdata.index('endstream', startstream)
86 if fdata[endstream-2:endstream] == '\r\n':
87 endstream -= 2
88 elif fdata[endstream-1] in ['\n', '\r']:
89 endstream -= 1
90 source = PdfTokens(fdata, endstream)
91 endit = source.multiple(2)
92 assert endit == 'endstream endobj'.split()
93 obj.Length = str(endstream-startstream)
94 obj._stream = fdata[startstream:endstream]
95 readstream = staticmethod(readstream)
96
97 def readarray(self, source, setobj=lambda x:None, original=None):
98 special = self.special
99 result = PdfArray()
100 setobj(result)
101
102 for value in source:
103 if value == ']':
104 break
105 if value in special:
106 value = special[value](source)
107 elif value == 'R':
108 generation = result.pop()
109 value = self.readindirect(result.pop(), generation)
110 result.append(value)
111 return result
112
113 def readdict(self, source, setobj=lambda x:None, original=None):
114 special = self.special
115 result = PdfDict()
116 setobj(result)
117
118 tok = source.next()
119 while tok != '>>':
120 assert tok.startswith('/'), (tok, source.multiple(10))
121 key = tok
122 value = source.next()
123 if value in special:
124 value = special[value](source)
125 tok = source.next()
126 else:
127 tok = source.next()
128 if value.isdigit() and tok.isdigit():
129 assert source.next() == 'R'
130 value = self.readindirect(value, tok)
131 tok = source.next()
132 result[key] = value
133
134 return result
135
136 def readxref(fdata):
137 startloc = fdata.rindex('startxref')
138 xrefinfo = list(PdfTokens(fdata, startloc, False))
139 assert len(xrefinfo) == 3, xrefinfo
140 assert xrefinfo[0] == 'startxref', xrefinfo[0]
141 assert xrefinfo[1].isdigit(), xrefinfo[1]
142 assert xrefinfo[2].rstrip() == '%%EOF', repr(xrefinfo[2])
143 return startloc, PdfTokens(fdata, int(xrefinfo[1]))
144 readxref = staticmethod(readxref)
145
146 def parsexref(self, source):
147 tok = source.next()
148 assert tok == 'xref', tok
149 while 1:
150 tok = source.next()
151 if tok == 'trailer':
152 break
153 startobj = int(tok)
154 for objnum in range(startobj, startobj + int(source.next())):
155 offset = int(source.next())
156 generation = int(source.next())
157 if source.next() == 'n':
158 objid = self.fdata, objnum, generation
159 objval = [offset, self.unresolved]
160 self.indirect_objects.setdefault(objid, objval)
161
162 pagename = PdfName.Page
163 pagesname = PdfName.Pages
164
165 def readpages(self, node):
166 # PDFs can have arbitrarily nested Pages/Page
167 # dictionary structures.
168 if node.Type == self.pagename:
169 return [node]
170 assert node.Type == self.pagesname, node.Type
171 result = []
172 for node in node.Kids:
173 result.extend(self.readpages(node))
174 return result
175
176 def __init__(self, fname=None, fdata=None, decompress=True):
177
178 if fname is not None:
179 assert fdata is None
180 # Allow reading preexisting streams like pyPdf
181 if hasattr(fname, 'read'):
182 fdata = fname.read()
183 else:
184 f = open(fname, 'rb')
185 fdata = f.read()
186 f.close()
187
188 assert fdata is not None
189 fdata = fdata.rstrip('\00')
190 self.private.fdata = fdata
191
192 self.private.indirect_objects = {}
193 self.private.special = {'<<': self.readdict, '[': self.readarray}
194
195 startloc, source = self.readxref(fdata)
196 self.parsexref(source)
197 assert source.next() == '<<'
198 self.update(self.readdict(source))
199 assert source.next() == 'startxref' and source.floc > startloc
200 self.private.pages = self.readpages(self.Root.Pages)
201 if decompress:
202 self.uncompress()
203
204 # For compatibility with pyPdf
205 self.private.numPages = len(self.pages)
206
207
208 # For compatibility with pyPdf
209 def getPage(self, pagenum):
210 return self.pages[pagenum]
211
212 def uncompress(self):
213 uncompress([x[1] for x in self.indirect_objects.itervalues()])
diff --git a/lib/pdfrw/pdftokens.py b/lib/pdfrw/pdftokens.py
deleted file mode 100644
index 04bd559..0000000
--- a/lib/pdfrw/pdftokens.py
+++ /dev/null
@@ -1,249 +0,0 @@
1# A part of pdfrw (pdfrw.googlecode.com)
2# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas
3# MIT license -- See LICENSE.txt for details
4
5'''
6A tokenizer for PDF streams.
7
8In general, documentation used was "PDF reference",
9sixth edition, for PDF version 1.7, dated November 2006.
10
11'''
12
13from __future__ import generators
14
15try:
16 set
17except NameError:
18 from sets import Set as set
19
20import re
21from pdfobjects import PdfString, PdfObject
22
23class _PrimitiveTokens(object):
24
25 # Table 3.1, page 50 of reference, defines whitespace
26 whitespaceset = set('\x00\t\n\f\r ')
27
28
29 # Text on page 50 defines delimiter characters
30 delimiterset = set('()<>{}[]/%')
31
32 # Coalesce contiguous whitespace into a single token
33 whitespace_pattern = '[%s]+' % ''.join(whitespaceset)
34
35 # In addition to the delimiters, we also use '\', which
36 # is special in some contexts in PDF.
37 delimiter_pattern = '\\\\|\\' + '|\\'.join(delimiterset)
38
39 # Dictionary delimiters are '<<' and '>>'. Look for
40 # these before the single variety.
41 dictdelim_pattern = r'\<\<|\>\>'
42
43 pattern = '(%s|%s|%s)' % (whitespace_pattern,
44 dictdelim_pattern, delimiter_pattern)
45 re_func = re.compile(pattern).finditer
46 del whitespace_pattern, dictdelim_pattern
47 del delimiter_pattern, pattern
48
49 def __init__(self, fdata):
50
51 class MyIterator(object):
52 def next():
53 if not tokens:
54 startloc = self.startloc
55 for match in next_match[0]:
56 start = match.start()
57 end = match.end()
58 tappend(fdata[start:end])
59 if start > startloc:
60 tappend(fdata[startloc:start])
61 self.startloc = end
62 break
63 else:
64 s = fdata[startloc:]
65 self.startloc = len(fdata)
66 if s:
67 tappend(s)
68 if not tokens:
69 raise StopIteration
70 return tpop()
71 next = staticmethod(next)
72
73 self.fdata = fdata
74 self.tokens = tokens = []
75 self.iterator = iterator = MyIterator()
76 self.next = iterator.next
77 self.next_match = next_match = [None]
78 tappend = tokens.append
79 tpop = tokens.pop
80
81 def setstart(self, startloc):
82 self.startloc = startloc
83 self.next_match[0] = self.re_func(self.fdata, startloc)
84
85 def __iter__(self):
86 return self.iterator
87
88 def coalesce(self, result):
89 ''' This function coalesces tokens together up until
90 the next delimiter or whitespace.
91 All of the coalesced tokens will either be non-matches,
92 or will be a matched backslash. We distinguish the
93 non-matches by the fact that next() will have left
94 a following match inside self.tokens for the actual match.
95 '''
96 tokens = self.tokens
97 whitespace = self.whitespaceset
98
99 # Optimized path for usual case -- regular data (not a name string),
100 # with no escape character, and followed by whitespace.
101
102 if tokens:
103 token = tokens.pop()
104 if token != '\\':
105 if token[0] not in whitespace:
106 tokens.append(token)
107 return
108 result.append(token)
109
110 # Non-optimized path. Either start of a name string received,
111 # or we just had one escape.
112
113 for token in self:
114 if tokens:
115 result.append(token)
116 token = tokens.pop()
117 if token != '\\':
118 if token[0] not in whitespace:
119 tokens.append(token)
120 return
121 result.append(token)
122
123
124 def floc(self):
125 return self.startloc - sum([len(x) for x in self.tokens])
126
127class PdfTokens(object):
128
129 def __init__(self, fdata, startloc=0, strip_comments=True):
130
131 def comment(token):
132 tokens = [token]
133 for token in primitive:
134 tokens.append(token)
135 if token[0] in whitespaceset and ('\n' in token or '\r' in token):
136 break
137 return not strip_comments and ''.join(tokens)
138
139 def single(token):
140 return token
141
142 def regular_string(token):
143 def escaped():
144 escaped = False
145 i = -2
146 while tokens[i] == '\\':
147 escaped = not escaped
148 i -= 1
149 return escaped
150
151 tokens = [token]
152 nestlevel = 1
153 for token in primitive:
154 tokens.append(token)
155 if token in '()' and not escaped():
156 nestlevel += token == '(' or -1
157 if not nestlevel:
158 break
159 else:
160 assert 0, "Unexpected end of token stream"
161 return PdfString(''.join(tokens))
162
163 def hex_string(token):
164 tokens = [token]
165 for token in primitive:
166 tokens.append(token)
167 if token == '>':
168 break
169 while tokens[-2] == '>>':
170 tokens.append(tokens.pop(-2))
171 return PdfString(''.join(tokens))
172
173 def normal_data(token):
174
175 # Obscure optimization -- we can get here with
176 # whitespace or regular character data. If we get
177 # here with whitespace, then there won't be an additional
178 # token queued up in the primitive object, otherwise there
179 # will...
180 if primitive_tokens: #if token[0] not in whitespaceset:
181 tokens = [token]
182 primitive.coalesce(tokens)
183 return PdfObject(''.join(tokens))
184
185 def name_string(token):
186 tokens = [token]
187 primitive.coalesce(tokens)
188 token = ''.join(tokens)
189 if '#' in token:
190 substrs = token.split('#')
191 substrs.reverse()
192 tokens = [substrs.pop()]
193 while substrs:
194 s = substrs.pop()
195 tokens.append(chr(int(s[:2], 16)))
196 tokens.append(s[2:])
197 token = ''.join(tokens)
198 return PdfObject(token)
199
200 def broken(token):
201 assert 0, token
202
203 dispatch = {
204 '(': regular_string,
205 ')': broken,
206 '<': hex_string,
207 '>': broken,
208 '[': single,
209 ']': single,
210 '{': single,
211 '}': single,
212 '/': name_string,
213 '%' : comment,
214 '<<': single,
215 '>>': single,
216 }.get
217
218 class MyIterator(object):
219 def next():
220 while not tokens:
221 token = primitive_next()
222 token = dispatch(token, normal_data)(token)
223 if token:
224 return token
225 return tokens.pop()
226 next = staticmethod(next)
227
228 self.primitive = primitive = _PrimitiveTokens(fdata)
229 self.setstart = primitive.setstart
230 primitive.setstart(startloc)
231 self.fdata = fdata
232 self.strip_comments = strip_comments
233 self.tokens = tokens = []
234 self.iterator = iterator = MyIterator()
235 self.next = iterator.next
236 primitive_next = primitive.next
237 primitive_tokens = primitive.tokens
238 whitespaceset = _PrimitiveTokens.whitespaceset
239
240 def floc(self):
241 return self.primitive.floc() - sum([len(x) for x in self.tokens])
242 floc = property(floc)
243
244 def __iter__(self):
245 return self.iterator
246
247 def multiple(self, count):
248 next = self.next
249 return [next() for i in range(count)]
diff --git a/lib/pdfrw/pdfwriter.py b/lib/pdfrw/pdfwriter.py
deleted file mode 100644
index c193843..0000000
--- a/lib/pdfrw/pdfwriter.py
+++ /dev/null
@@ -1,234 +0,0 @@
1#!/usr/bin/env python
2
3# A part of pdfrw (pdfrw.googlecode.com)
4# Copyright (C) 2006-2009 Patrick Maupin, Austin, Texas
5# MIT license -- See LICENSE.txt for details
6
7'''
8The PdfWriter class writes an entire PDF file out to disk.
9
10The writing process is not at all optimized or organized.
11
12An instance of the PdfWriter class has two methods:
13 addpage(page)
14and
15 write(fname)
16
17addpage() assumes that the pages are part of a valid
18tree/forest of PDF objects.
19'''
20
21try:
22 set
23except NameError:
24 from sets import Set as set
25
26from pdfobjects import PdfName, PdfArray, PdfDict, IndirectPdfDict, PdfObject, PdfString
27from pdfcompress import compress
28
29debug = False
30
31class FormatObjects(object):
32 ''' FormatObjects performs the actual formatting and disk write.
33 '''
34
35 def add(self, obj, visited):
36 ''' Add an object to our list, if it's an indirect
37 object. Just format it if not.
38 '''
39 # Can't hash dicts, so just hash the object ID
40 objid = id(obj)
41
42 # Automatically set stream objects to indirect
43 if isinstance(obj, PdfDict):
44 indirect = obj.indirect or (obj.stream is not None)
45 else:
46 indirect = getattr(obj, 'indirect', False)
47
48 if not indirect:
49 assert objid not in visited, \
50 'Circular reference encountered in non-indirect object %s' % repr(obj)
51 visited.add(objid)
52 result = self.format_obj(obj, visited)
53 visited.remove(objid)
54 return result
55
56 objnum = self.indirect_dict.get(objid)
57
58 # If we haven't seen the object yet, we need to
59 # add it to the indirect object list.
60 if objnum is None:
61 objlist = self.objlist
62 objnum = len(objlist) + 1
63 if debug:
64 print ' Object', objnum, '\r',
65 objlist.append(None)
66 self.indirect_dict[objid] = objnum
67 objlist[objnum-1] = self.format_obj(obj)
68 return '%s 0 R' % objnum
69
70 def format_array(myarray, formatter):
71 # Format array data into semi-readable ASCII
72 if sum([len(x) for x in myarray]) <= 70:
73 return formatter % ' '.join(myarray)
74 bigarray = []
75 count = 1000000
76 for x in myarray:
77 lenx = len(x)
78 if lenx + count > 70:
79 subarray = []
80 bigarray.append(subarray)
81 count = 0
82 count += lenx + 1
83 subarray.append(x)
84 return formatter % '\n '.join([' '.join(x) for x in bigarray])
85 format_array = staticmethod(format_array)
86
87 def format_obj(self, obj, visited=None):
88 ''' format PDF object data into semi-readable ASCII.
89 May mutually recurse with add() -- add() will
90 return references for indirect objects, and add
91 the indirect object to the list.
92 '''
93 if visited is None:
94 visited = set()
95 if isinstance(obj, PdfArray):
96 myarray = [self.add(x, visited) for x in obj]
97 return self.format_array(myarray, '[%s]')
98 elif isinstance(obj, PdfDict):
99 if self.compress and obj.stream:
100 compress([obj])
101 myarray = []
102 # Jython 2.2.1 has a bug which segfaults when
103 # sorting subclassed strings, so we un-subclass them.
104 dictkeys = [str(x) for x in obj.iterkeys()]
105 dictkeys.sort()
106 for key in dictkeys:
107 myarray.append(key)
108 myarray.append(self.add(obj[key], visited))
109 result = self.format_array(myarray, '<<%s>>')
110 stream = obj.stream
111 if stream is not None:
112 result = '%s\nstream\n%s\nendstream' % (result, stream)
113 return result
114 elif isinstance(obj, basestring) and not hasattr(obj, 'indirect'):
115 return PdfString.encode(obj)
116 else:
117 return str(obj)
118
119 def dump(cls, f, trailer, version='1.3', compress=True):
120 self = cls()
121 self.compress = compress
122 self.indirect_dict = {}
123 self.objlist = []
124
125 # The first format of trailer gets all the information,
126 # but we throw away the actual trailer formatting.
127 self.format_obj(trailer)
128 # Now we know the size, so we update the trailer dict
129 # and get the formatted data.
130 trailer.Size = PdfObject(len(self.objlist) + 1)
131 trailer = self.format_obj(trailer)
132
133 # Now we have all the pieces to write out to the file.
134 # Keep careful track of the counts while we do it so
135 # we can correctly build the cross-reference.
136
137 header = '%%PDF-%s\n%%\xe2\xe3\xcf\xd3\n' % version
138 f.write(header)
139 offset = len(header)
140 offsets = [(0, 65535, 'f')]
141
142 for i, x in enumerate(self.objlist):
143 objstr = '%s 0 obj\n%s\nendobj\n' % (i + 1, x)
144 offsets.append((offset, 0, 'n'))
145 offset += len(objstr)
146 f.write(objstr)
147
148 f.write('xref\n0 %s\n' % len(offsets))
149 for x in offsets:
150 f.write('%010d %05d %s\r\n' % x)
151 f.write('trailer\n\n%s\nstartxref\n%s\n%%%%EOF\n' % (trailer, offset))
152 dump = classmethod(dump)
153
154class PdfWriter(object):
155
156 _trailer = None
157
158 def __init__(self, version='1.3', compress=True):
159 self.pagearray = PdfArray()
160 self.compress = compress
161 self.version = version
162
163 def addpage(self, page):
164 self._trailer = None
165 assert page.Type == PdfName.Page
166 inheritable = page.inheritable # searches for resources
167 self.pagearray.append(
168 IndirectPdfDict(
169 page,
170 Resources = inheritable.Resources,
171 MediaBox = inheritable.MediaBox,
172 CropBox = inheritable.CropBox,
173 Rotate = inheritable.Rotate,
174 )
175 )
176 return self
177
178 addPage = addpage # for compatibility with pyPdf
179
180 def addpages(self, pagelist):
181 for page in pagelist:
182 self.addpage(page)
183 return self
184
185 def _get_trailer(self):
186 trailer = self._trailer
187 if trailer is not None:
188 return trailer
189
190 # Create the basic object structure of the PDF file
191 trailer = PdfDict(
192 Root = IndirectPdfDict(
193 Type = PdfName.Catalog,
194 Pages = IndirectPdfDict(
195 Type = PdfName.Pages,
196 Count = PdfObject(len(self.pagearray)),
197 Kids = self.pagearray
198 )
199 )
200 )
201 # Make all the pages point back to the page dictionary
202 pagedict = trailer.Root.Pages
203 for page in pagedict.Kids:
204 page.Parent = pagedict
205 self._trailer = trailer
206 return trailer
207
208 def _set_trailer(self, trailer):
209 self._trailer = trailer
210
211 trailer = property(_get_trailer, _set_trailer)
212
213 def write(self, fname, trailer=None):
214 trailer = trailer or self.trailer
215
216 # Dump the data. We either have a filename or a preexisting
217 # file object.
218 preexisting = hasattr(fname, 'write')
219 f = preexisting and fname or open(fname, 'wb')
220 FormatObjects.dump(f, trailer, self.version, self.compress)
221 if not preexisting:
222 f.close()
223
224if __name__ == '__main__':
225 debug = True
226 import pdfreader
227 x = pdfreader.PdfReader('source.pdf')
228 y = PdfWriter()
229 for i, page in enumerate(x.pages):
230 print ' Adding page', i+1, '\r',
231 y.addpage(page)
232 print
233 y.write('result.pdf')
234 print
diff --git a/lib/tarfile/__init__.py b/lib/tarfile/__init__.py
deleted file mode 100644
index 8b13789..0000000
--- a/lib/tarfile/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
1
diff --git a/lib/tarfile/tarfile.py b/lib/tarfile/tarfile.py
deleted file mode 100755
index 37c9b92..0000000
--- a/lib/tarfile/tarfile.py
+++ /dev/null
@@ -1,2594 +0,0 @@
1#! /usr/bin/python2.7
2# -*- coding: iso-8859-1 -*-
3#-------------------------------------------------------------------
4# tarfile.py
5#-------------------------------------------------------------------
6# Copyright (C) 2002 Lars Gustäbel <lars@gustaebel.de>
7# All rights reserved.
8#
9# Permission is hereby granted, free of charge, to any person
10# obtaining a copy of this software and associated documentation
11# files (the "Software"), to deal in the Software without
12# restriction, including without limitation the rights to use,
13# copy, modify, merge, publish, distribute, sublicense, and/or sell
14# copies of the Software, and to permit persons to whom the
15# Software is furnished to do so, subject to the following
16# conditions:
17#
18# The above copyright notice and this permission notice shall be
19# included in all copies or substantial portions of the Software.
20#
21# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28# OTHER DEALINGS IN THE SOFTWARE.
29#
30"""Read from and write to tar format archives.
31"""
32
33__version__ = "$Revision$"
34# $Source$
35
36version = "0.9.0"
37__author__ = "Lars Gustäbel (lars@gustaebel.de)"
38__date__ = "$Date$"
39__cvsid__ = "$Id$"
40__credits__ = "Gustavo Niemeyer, Niels Gustäbel, Richard Townsend."
41
42#---------
43# Imports
44#---------
45import sys
46import os
47import shutil
48import stat
49import errno
50import time
51import struct
52import copy
53import re
54import operator
55
56try:
57 import grp, pwd
58except ImportError:
59 grp = pwd = None
60
61# from tarfile import *
62__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"]
63
64#---------------------------------------------------------
65# tar constants
66#---------------------------------------------------------
67NUL = "\0" # the null character
68BLOCKSIZE = 512 # length of processing blocks
69RECORDSIZE = BLOCKSIZE * 20 # length of records
70GNU_MAGIC = "ustar \0" # magic gnu tar string
71POSIX_MAGIC = "ustar\x0000" # magic posix tar string
72
73LENGTH_NAME = 100 # maximum length of a filename
74LENGTH_LINK = 100 # maximum length of a linkname
75LENGTH_PREFIX = 155 # maximum length of the prefix field
76
77REGTYPE = "0" # regular file
78AREGTYPE = "\0" # regular file
79LNKTYPE = "1" # link (inside tarfile)
80SYMTYPE = "2" # symbolic link
81CHRTYPE = "3" # character special device
82BLKTYPE = "4" # block special device
83DIRTYPE = "5" # directory
84FIFOTYPE = "6" # fifo special device
85CONTTYPE = "7" # contiguous file
86
87GNUTYPE_LONGNAME = "L" # GNU tar longname
88GNUTYPE_LONGLINK = "K" # GNU tar longlink
89GNUTYPE_SPARSE = "S" # GNU tar sparse file
90
91XHDTYPE = "x" # POSIX.1-2001 extended header
92XGLTYPE = "g" # POSIX.1-2001 global header
93SOLARIS_XHDTYPE = "X" # Solaris extended header
94
95USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format
96GNU_FORMAT = 1 # GNU tar format
97PAX_FORMAT = 2 # POSIX.1-2001 (pax) format
98DEFAULT_FORMAT = GNU_FORMAT
99
100#---------------------------------------------------------
101# tarfile constants
102#---------------------------------------------------------
103# File types that tarfile supports:
104SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE,
105 SYMTYPE, DIRTYPE, FIFOTYPE,
106 CONTTYPE, CHRTYPE, BLKTYPE,
107 GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
108 GNUTYPE_SPARSE)
109
110# File types that will be treated as a regular file.
111REGULAR_TYPES = (REGTYPE, AREGTYPE,
112 CONTTYPE, GNUTYPE_SPARSE)
113
114# File types that are part of the GNU tar format.
115GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
116 GNUTYPE_SPARSE)
117
118# Fields from a pax header that override a TarInfo attribute.
119PAX_FIELDS = ("path", "linkpath", "size", "mtime",
120 "uid", "gid", "uname", "gname")
121
122# Fields in a pax header that are numbers, all other fields
123# are treated as strings.
124PAX_NUMBER_FIELDS = {
125 "atime": float,
126 "ctime": float,
127 "mtime": float,
128 "uid": int,
129 "gid": int,
130 "size": int
131}
132
133#---------------------------------------------------------
134# Bits used in the mode field, values in octal.
135#---------------------------------------------------------
136S_IFLNK = 0120000 # symbolic link
137S_IFREG = 0100000 # regular file
138S_IFBLK = 0060000 # block device
139S_IFDIR = 0040000 # directory
140S_IFCHR = 0020000 # character device
141S_IFIFO = 0010000 # fifo
142
143TSUID = 04000 # set UID on execution
144TSGID = 02000 # set GID on execution
145TSVTX = 01000 # reserved
146
147TUREAD = 0400 # read by owner
148TUWRITE = 0200 # write by owner
149TUEXEC = 0100 # execute/search by owner
150TGREAD = 0040 # read by group
151TGWRITE = 0020 # write by group
152TGEXEC = 0010 # execute/search by group
153TOREAD = 0004 # read by other
154TOWRITE = 0002 # write by other
155TOEXEC = 0001 # execute/search by other
156
157#---------------------------------------------------------
158# initialization
159#---------------------------------------------------------
160ENCODING = sys.getfilesystemencoding()
161if ENCODING is None:
162 ENCODING = sys.getdefaultencoding()
163
164#---------------------------------------------------------
165# Some useful functions
166#---------------------------------------------------------
167
168def stn(s, length):
169 """Convert a python string to a null-terminated string buffer.
170 """
171 return s[:length] + (length - len(s)) * NUL
172
173def nts(s):
174 """Convert a null-terminated string field to a python string.
175 """
176 # Use the string up to the first null char.
177 p = s.find("\0")
178 if p == -1:
179 return s
180 return s[:p]
181
182def nti(s):
183 """Convert a number field to a python number.
184 """
185 # There are two possible encodings for a number field, see
186 # itn() below.
187 if s[0] != chr(0200):
188 try:
189 n = int(nts(s) or "0", 8)
190 except ValueError:
191 raise InvalidHeaderError("invalid header")
192 else:
193 n = 0L
194 for i in xrange(len(s) - 1):
195 n <<= 8
196 n += ord(s[i + 1])
197 return n
198
199def itn(n, digits=8, format=DEFAULT_FORMAT):
200 """Convert a python number to a number field.
201 """
202 # POSIX 1003.1-1988 requires numbers to be encoded as a string of
203 # octal digits followed by a null-byte, this allows values up to
204 # (8**(digits-1))-1. GNU tar allows storing numbers greater than
205 # that if necessary. A leading 0200 byte indicates this particular
206 # encoding, the following digits-1 bytes are a big-endian
207 # representation. This allows values up to (256**(digits-1))-1.
208 if 0 <= n < 8 ** (digits - 1):
209 s = "%0*o" % (digits - 1, n) + NUL
210 else:
211 if format != GNU_FORMAT or n >= 256 ** (digits - 1):
212 raise ValueError("overflow in number field")
213
214 if n < 0:
215 # XXX We mimic GNU tar's behaviour with negative numbers,
216 # this could raise OverflowError.
217 n = struct.unpack("L", struct.pack("l", n))[0]
218
219 s = ""
220 for i in xrange(digits - 1):
221 s = chr(n & 0377) + s
222 n >>= 8
223 s = chr(0200) + s
224 return s
225
226def uts(s, encoding, errors):
227 """Convert a unicode object to a string.
228 """
229 if errors == "utf-8":
230 # An extra error handler similar to the -o invalid=UTF-8 option
231 # in POSIX.1-2001. Replace untranslatable characters with their
232 # UTF-8 representation.
233 try:
234 return s.encode(encoding, "strict")
235 except UnicodeEncodeError:
236 x = []
237 for c in s:
238 try:
239 x.append(c.encode(encoding, "strict"))
240 except UnicodeEncodeError:
241 x.append(c.encode("utf8"))
242 return "".join(x)
243 else:
244 return s.encode(encoding, errors)
245
246def calc_chksums(buf):
247 """Calculate the checksum for a member's header by summing up all
248 characters except for the chksum field which is treated as if
249 it was filled with spaces. According to the GNU tar sources,
250 some tars (Sun and NeXT) calculate chksum with signed char,
251 which will be different if there are chars in the buffer with
252 the high bit set. So we calculate two checksums, unsigned and
253 signed.
254 """
255 unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
256 signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
257 return unsigned_chksum, signed_chksum
258
259def copyfileobj(src, dst, length=None):
260 """Copy length bytes from fileobj src to fileobj dst.
261 If length is None, copy the entire content.
262 """
263 if length == 0:
264 return
265 if length is None:
266 shutil.copyfileobj(src, dst)
267 return
268
269 BUFSIZE = 16 * 1024
270 blocks, remainder = divmod(length, BUFSIZE)
271 for b in xrange(blocks):
272 buf = src.read(BUFSIZE)
273 if len(buf) < BUFSIZE:
274 raise IOError("end of file reached")
275 dst.write(buf)
276
277 if remainder != 0:
278 buf = src.read(remainder)
279 if len(buf) < remainder:
280 raise IOError("end of file reached")
281 dst.write(buf)
282 return
283
284filemode_table = (
285 ((S_IFLNK, "l"),
286 (S_IFREG, "-"),
287 (S_IFBLK, "b"),
288 (S_IFDIR, "d"),
289 (S_IFCHR, "c"),
290 (S_IFIFO, "p")),
291
292 ((TUREAD, "r"),),
293 ((TUWRITE, "w"),),
294 ((TUEXEC|TSUID, "s"),
295 (TSUID, "S"),
296 (TUEXEC, "x")),
297
298 ((TGREAD, "r"),),
299 ((TGWRITE, "w"),),
300 ((TGEXEC|TSGID, "s"),
301 (TSGID, "S"),
302 (TGEXEC, "x")),
303
304 ((TOREAD, "r"),),
305 ((TOWRITE, "w"),),
306 ((TOEXEC|TSVTX, "t"),
307 (TSVTX, "T"),
308 (TOEXEC, "x"))
309)
310
311def filemode(mode):
312 """Convert a file's mode to a string of the form
313 -rwxrwxrwx.
314 Used by TarFile.list()
315 """
316 perm = []
317 for table in filemode_table:
318 for bit, char in table:
319 if mode & bit == bit:
320 perm.append(char)
321 break
322 else:
323 perm.append("-")
324 return "".join(perm)
325
326class TarError(Exception):
327 """Base exception."""
328 pass
329class ExtractError(TarError):
330 """General exception for extract errors."""
331 pass
332class ReadError(TarError):
333 """Exception for unreadble tar archives."""
334 pass
335class CompressionError(TarError):
336 """Exception for unavailable compression methods."""
337 pass
338class StreamError(TarError):
339 """Exception for unsupported operations on stream-like TarFiles."""
340 pass
341class HeaderError(TarError):
342 """Base exception for header errors."""
343 pass
344class EmptyHeaderError(HeaderError):
345 """Exception for empty headers."""
346 pass
347class TruncatedHeaderError(HeaderError):
348 """Exception for truncated headers."""
349 pass
350class EOFHeaderError(HeaderError):
351 """Exception for end of file headers."""
352 pass
353class InvalidHeaderError(HeaderError):
354 """Exception for invalid headers."""
355 pass
356class SubsequentHeaderError(HeaderError):
357 """Exception for missing and invalid extended headers."""
358 pass
359
360#---------------------------
361# internal stream interface
362#---------------------------
363class _LowLevelFile:
364 """Low-level file object. Supports reading and writing.
365 It is used instead of a regular file object for streaming
366 access.
367 """
368
369 def __init__(self, name, mode):
370 mode = {
371 "r": os.O_RDONLY,
372 "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
373 }[mode]
374 if hasattr(os, "O_BINARY"):
375 mode |= os.O_BINARY
376 self.fd = os.open(name, mode, 0666)
377
378 def close(self):
379 os.close(self.fd)
380
381 def read(self, size):
382 return os.read(self.fd, size)
383
384 def write(self, s):
385 os.write(self.fd, s)
386
387class _Stream:
388 """Class that serves as an adapter between TarFile and
389 a stream-like object. The stream-like object only
390 needs to have a read() or write() method and is accessed
391 blockwise. Use of gzip or bzip2 compression is possible.
392 A stream-like object could be for example: sys.stdin,
393 sys.stdout, a socket, a tape device etc.
394
395 _Stream is intended to be used only internally.
396 """
397
398 def __init__(self, name, mode, comptype, fileobj, bufsize):
399 """Construct a _Stream object.
400 """
401 self._extfileobj = True
402 if fileobj is None:
403 fileobj = _LowLevelFile(name, mode)
404 self._extfileobj = False
405
406 if comptype == '*':
407 # Enable transparent compression detection for the
408 # stream interface
409 fileobj = _StreamProxy(fileobj)
410 comptype = fileobj.getcomptype()
411
412 self.name = name or ""
413 self.mode = mode
414 self.comptype = comptype
415 self.fileobj = fileobj
416 self.bufsize = bufsize
417 self.buf = ""
418 self.pos = 0L
419 self.closed = False
420
421 if comptype == "gz":
422 try:
423 import zlib
424 except ImportError:
425 raise CompressionError("zlib module is not available")
426 self.zlib = zlib
427 self.crc = zlib.crc32("") & 0xffffffffL
428 if mode == "r":
429 self._init_read_gz()
430 else:
431 self._init_write_gz()
432
433 if comptype == "bz2":
434 try:
435 import bz2
436 except ImportError:
437 raise CompressionError("bz2 module is not available")
438 if mode == "r":
439 self.dbuf = ""
440 self.cmp = bz2.BZ2Decompressor()
441 else:
442 self.cmp = bz2.BZ2Compressor()
443
444 def __del__(self):
445 if hasattr(self, "closed") and not self.closed:
446 self.close()
447
448 def _init_write_gz(self):
449 """Initialize for writing with gzip compression.
450 """
451 self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED,
452 -self.zlib.MAX_WBITS,
453 self.zlib.DEF_MEM_LEVEL,
454 0)
455 timestamp = struct.pack("<L", long(time.time()))
456 self.__write("\037\213\010\010%s\002\377" % timestamp)
457 if self.name.endswith(".gz"):
458 self.name = self.name[:-3]
459 self.__write(self.name + NUL)
460
461 def write(self, s):
462 """Write string s to the stream.
463 """
464 if self.comptype == "gz":
465 self.crc = self.zlib.crc32(s, self.crc) & 0xffffffffL
466 self.pos += len(s)
467 if self.comptype != "tar":
468 s = self.cmp.compress(s)
469 self.__write(s)
470
471 def __write(self, s):
472 """Write string s to the stream if a whole new block
473 is ready to be written.
474 """
475 self.buf += s
476 while len(self.buf) > self.bufsize:
477 self.fileobj.write(self.buf[:self.bufsize])
478 self.buf = self.buf[self.bufsize:]
479
480 def close(self):
481 """Close the _Stream object. No operation should be
482 done on it afterwards.
483 """
484 if self.closed:
485 return
486
487 if self.mode == "w" and self.comptype != "tar":
488 self.buf += self.cmp.flush()
489
490 if self.mode == "w" and self.buf:
491 self.fileobj.write(self.buf)
492 self.buf = ""
493 if self.comptype == "gz":
494 # The native zlib crc is an unsigned 32-bit integer, but
495 # the Python wrapper implicitly casts that to a signed C
496 # long. So, on a 32-bit box self.crc may "look negative",
497 # while the same crc on a 64-bit box may "look positive".
498 # To avoid irksome warnings from the `struct` module, force
499 # it to look positive on all boxes.
500 self.fileobj.write(struct.pack("<L", self.crc & 0xffffffffL))
501 self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFFL))
502
503 if not self._extfileobj:
504 self.fileobj.close()
505
506 self.closed = True
507
508 def _init_read_gz(self):
509 """Initialize for reading a gzip compressed fileobj.
510 """
511 self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS)
512 self.dbuf = ""
513
514 # taken from gzip.GzipFile with some alterations
515 if self.__read(2) != "\037\213":
516 raise ReadError("not a gzip file")
517 if self.__read(1) != "\010":
518 raise CompressionError("unsupported compression method")
519
520 flag = ord(self.__read(1))
521 self.__read(6)
522
523 if flag & 4:
524 xlen = ord(self.__read(1)) + 256 * ord(self.__read(1))
525 self.read(xlen)
526 if flag & 8:
527 while True:
528 s = self.__read(1)
529 if not s or s == NUL:
530 break
531 if flag & 16:
532 while True:
533 s = self.__read(1)
534 if not s or s == NUL:
535 break
536 if flag & 2:
537 self.__read(2)
538
539 def tell(self):
540 """Return the stream's file pointer position.
541 """
542 return self.pos
543
544 def seek(self, pos=0):
545 """Set the stream's file pointer to pos. Negative seeking
546 is forbidden.
547 """
548 if pos - self.pos >= 0:
549 blocks, remainder = divmod(pos - self.pos, self.bufsize)
550 for i in xrange(blocks):
551 self.read(self.bufsize)
552 self.read(remainder)
553 else:
554 raise StreamError("seeking backwards is not allowed")
555 return self.pos
556
557 def read(self, size=None):
558 """Return the next size number of bytes from the stream.
559 If size is not defined, return all bytes of the stream
560 up to EOF.
561 """
562 if size is None:
563 t = []
564 while True:
565 buf = self._read(self.bufsize)
566 if not buf:
567 break
568 t.append(buf)
569 buf = "".join(t)
570 else:
571 buf = self._read(size)
572 self.pos += len(buf)
573 return buf
574
575 def _read(self, size):
576 """Return size bytes from the stream.
577 """
578 if self.comptype == "tar":
579 return self.__read(size)
580
581 c = len(self.dbuf)
582 t = [self.dbuf]
583 while c < size:
584 buf = self.__read(self.bufsize)
585 if not buf:
586 break
587 try:
588 buf = self.cmp.decompress(buf)
589 except IOError:
590 raise ReadError("invalid compressed data")
591 t.append(buf)
592 c += len(buf)
593 t = "".join(t)
594 self.dbuf = t[size:]
595 return t[:size]
596
597 def __read(self, size):
598 """Return size bytes from stream. If internal buffer is empty,
599 read another block from the stream.
600 """
601 c = len(self.buf)
602 t = [self.buf]
603 while c < size:
604 buf = self.fileobj.read(self.bufsize)
605 if not buf:
606 break
607 t.append(buf)
608 c += len(buf)
609 t = "".join(t)
610 self.buf = t[size:]
611 return t[:size]
612# class _Stream
613
614class _StreamProxy(object):
615 """Small proxy class that enables transparent compression
616 detection for the Stream interface (mode 'r|*').
617 """
618
619 def __init__(self, fileobj):
620 self.fileobj = fileobj
621 self.buf = self.fileobj.read(BLOCKSIZE)
622
623 def read(self, size):
624 self.read = self.fileobj.read
625 return self.buf
626
627 def getcomptype(self):
628 if self.buf.startswith("\037\213\010"):
629 return "gz"
630 if self.buf.startswith("BZh91"):
631 return "bz2"
632 return "tar"
633
634 def close(self):
635 self.fileobj.close()
636# class StreamProxy
637
638class _BZ2Proxy(object):
639 """Small proxy class that enables external file object
640 support for "r:bz2" and "w:bz2" modes. This is actually
641 a workaround for a limitation in bz2 module's BZ2File
642 class which (unlike gzip.GzipFile) has no support for
643 a file object argument.
644 """
645
646 blocksize = 16 * 1024
647
648 def __init__(self, fileobj, mode):
649 self.fileobj = fileobj
650 self.mode = mode
651 self.name = getattr(self.fileobj, "name", None)
652 self.init()
653
654 def init(self):
655 import bz2
656 self.pos = 0
657 if self.mode == "r":
658 self.bz2obj = bz2.BZ2Decompressor()
659 self.fileobj.seek(0)
660 self.buf = ""
661 else:
662 self.bz2obj = bz2.BZ2Compressor()
663
664 def read(self, size):
665 b = [self.buf]
666 x = len(self.buf)
667 while x < size:
668 raw = self.fileobj.read(self.blocksize)
669 if not raw:
670 break
671 data = self.bz2obj.decompress(raw)
672 b.append(data)
673 x += len(data)
674 self.buf = "".join(b)
675
676 buf = self.buf[:size]
677 self.buf = self.buf[size:]
678 self.pos += len(buf)
679 return buf
680
681 def seek(self, pos):
682 if pos < self.pos:
683 self.init()
684 self.read(pos - self.pos)
685
686 def tell(self):
687 return self.pos
688
689 def write(self, data):
690 self.pos += len(data)
691 raw = self.bz2obj.compress(data)
692 self.fileobj.write(raw)
693
694 def close(self):
695 if self.mode == "w":
696 raw = self.bz2obj.flush()
697 self.fileobj.write(raw)
698# class _BZ2Proxy
699
700#------------------------
701# Extraction file object
702#------------------------
703class _FileInFile(object):
704 """A thin wrapper around an existing file object that
705 provides a part of its data as an individual file
706 object.
707 """
708
709 def __init__(self, fileobj, offset, size, sparse=None):
710 self.fileobj = fileobj
711 self.offset = offset
712 self.size = size
713 self.sparse = sparse
714 self.position = 0
715
716 def tell(self):
717 """Return the current file position.
718 """
719 return self.position
720
721 def seek(self, position):
722 """Seek to a position in the file.
723 """
724 self.position = position
725
726 def read(self, size=None):
727 """Read data from the file.
728 """
729 if size is None:
730 size = self.size - self.position
731 else:
732 size = min(size, self.size - self.position)
733
734 if self.sparse is None:
735 return self.readnormal(size)
736 else:
737 return self.readsparse(size)
738
739 def readnormal(self, size):
740 """Read operation for regular files.
741 """
742 self.fileobj.seek(self.offset + self.position)
743 self.position += size
744 return self.fileobj.read(size)
745
746 def readsparse(self, size):
747 """Read operation for sparse files.
748 """
749 data = []
750 while size > 0:
751 buf = self.readsparsesection(size)
752 if not buf:
753 break
754 size -= len(buf)
755 data.append(buf)
756 return "".join(data)
757
758 def readsparsesection(self, size):
759 """Read a single section of a sparse file.
760 """
761 section = self.sparse.find(self.position)
762
763 if section is None:
764 return ""
765
766 size = min(size, section.offset + section.size - self.position)
767
768 if isinstance(section, _data):
769 realpos = section.realpos + self.position - section.offset
770 self.fileobj.seek(self.offset + realpos)
771 self.position += size
772 return self.fileobj.read(size)
773 else:
774 self.position += size
775 return NUL * size
776#class _FileInFile
777
778
779class ExFileObject(object):
780 """File-like object for reading an archive member.
781 Is returned by TarFile.extractfile().
782 """
783 blocksize = 1024
784
785 def __init__(self, tarfile, tarinfo):
786 self.fileobj = _FileInFile(tarfile.fileobj,
787 tarinfo.offset_data,
788 tarinfo.size,
789 getattr(tarinfo, "sparse", None))
790 self.name = tarinfo.name
791 self.mode = "r"
792 self.closed = False
793 self.size = tarinfo.size
794
795 self.position = 0
796 self.buffer = ""
797
798 def read(self, size=None):
799 """Read at most size bytes from the file. If size is not
800 present or None, read all data until EOF is reached.
801 """
802 if self.closed:
803 raise ValueError("I/O operation on closed file")
804
805 buf = ""
806 if self.buffer:
807 if size is None:
808 buf = self.buffer
809 self.buffer = ""
810 else:
811 buf = self.buffer[:size]
812 self.buffer = self.buffer[size:]
813
814 if size is None:
815 buf += self.fileobj.read()
816 else:
817 buf += self.fileobj.read(size - len(buf))
818
819 self.position += len(buf)
820 return buf
821
822 def readline(self, size=-1):
823 """Read one entire line from the file. If size is present
824 and non-negative, return a string with at most that
825 size, which may be an incomplete line.
826 """
827 if self.closed:
828 raise ValueError("I/O operation on closed file")
829
830 if "\n" in self.buffer:
831 pos = self.buffer.find("\n") + 1
832 else:
833 buffers = [self.buffer]
834 while True:
835 buf = self.fileobj.read(self.blocksize)
836 buffers.append(buf)
837 if not buf or "\n" in buf:
838 self.buffer = "".join(buffers)
839 pos = self.buffer.find("\n") + 1
840 if pos == 0:
841 # no newline found.
842 pos = len(self.buffer)
843 break
844
845 if size != -1:
846 pos = min(size, pos)
847
848 buf = self.buffer[:pos]
849 self.buffer = self.buffer[pos:]
850 self.position += len(buf)
851 return buf
852
853 def readlines(self):
854 """Return a list with all remaining lines.
855 """
856 result = []
857 while True:
858 line = self.readline()
859 if not line: break
860 result.append(line)
861 return result
862
863 def tell(self):
864 """Return the current file position.
865 """
866 if self.closed:
867 raise ValueError("I/O operation on closed file")
868
869 return self.position
870
871 def seek(self, pos, whence=os.SEEK_SET):
872 """Seek to a position in the file.
873 """
874 if self.closed:
875 raise ValueError("I/O operation on closed file")
876
877 if whence == os.SEEK_SET:
878 self.position = min(max(pos, 0), self.size)
879 elif whence == os.SEEK_CUR:
880 if pos < 0:
881 self.position = max(self.position + pos, 0)
882 else:
883 self.position = min(self.position + pos, self.size)
884 elif whence == os.SEEK_END:
885 self.position = max(min(self.size + pos, self.size), 0)
886 else:
887 raise ValueError("Invalid argument")
888
889 self.buffer = ""
890 self.fileobj.seek(self.position)
891
892 def close(self):
893 """Close the file object.
894 """
895 self.closed = True
896
897 def __iter__(self):
898 """Get an iterator over the file's lines.
899 """
900 while True:
901 line = self.readline()
902 if not line:
903 break
904 yield line
905#class ExFileObject
906
907#------------------
908# Exported Classes
909#------------------
910class TarInfo(object):
911 """Informational class which holds the details about an
912 archive member given by a tar header block.
913 TarInfo objects are returned by TarFile.getmember(),
914 TarFile.getmembers() and TarFile.gettarinfo() and are
915 usually created internally.
916 """
917
918 def __init__(self, name=""):
919 """Construct a TarInfo object. name is the optional name
920 of the member.
921 """
922 self.name = name # member name
923 self.mode = 0644 # file permissions
924 self.uid = 0 # user id
925 self.gid = 0 # group id
926 self.size = 0 # file size
927 self.mtime = 0 # modification time
928 self.chksum = 0 # header checksum
929 self.type = REGTYPE # member type
930 self.linkname = "" # link name
931 self.uname = "" # user name
932 self.gname = "" # group name
933 self.devmajor = 0 # device major number
934 self.devminor = 0 # device minor number
935
936 self.offset = 0 # the tar header starts here
937 self.offset_data = 0 # the file's data starts here
938
939 self.pax_headers = {} # pax header information
940
941 # In pax headers the "name" and "linkname" field are called
942 # "path" and "linkpath".
943 def _getpath(self):
944 return self.name
945 def _setpath(self, name):
946 self.name = name
947 path = property(_getpath, _setpath)
948
949 def _getlinkpath(self):
950 return self.linkname
951 def _setlinkpath(self, linkname):
952 self.linkname = linkname
953 linkpath = property(_getlinkpath, _setlinkpath)
954
955 def __repr__(self):
956 return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self))
957
958 def get_info(self, encoding, errors):
959 """Return the TarInfo's attributes as a dictionary.
960 """
961 info = {
962 "name": self.name,
963 "mode": self.mode & 07777,
964 "uid": self.uid,
965 "gid": self.gid,
966 "size": self.size,
967 "mtime": self.mtime,
968 "chksum": self.chksum,
969 "type": self.type,
970 "linkname": self.linkname,
971 "uname": self.uname,
972 "gname": self.gname,
973 "devmajor": self.devmajor,
974 "devminor": self.devminor
975 }
976
977 if info["type"] == DIRTYPE and not info["name"].endswith("/"):
978 info["name"] += "/"
979
980 for key in ("name", "linkname", "uname", "gname"):
981 if type(info[key]) is unicode:
982 info[key] = info[key].encode(encoding, errors)
983
984 return info
985
986 def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="strict"):
987 """Return a tar header as a string of 512 byte blocks.
988 """
989 info = self.get_info(encoding, errors)
990
991 if format == USTAR_FORMAT:
992 return self.create_ustar_header(info)
993 elif format == GNU_FORMAT:
994 return self.create_gnu_header(info)
995 elif format == PAX_FORMAT:
996 return self.create_pax_header(info, encoding, errors)
997 else:
998 raise ValueError("invalid format")
999
1000 def create_ustar_header(self, info):
1001 """Return the object as a ustar header block.
1002 """
1003 info["magic"] = POSIX_MAGIC
1004
1005 if len(info["linkname"]) > LENGTH_LINK:
1006 raise ValueError("linkname is too long")
1007
1008 if len(info["name"]) > LENGTH_NAME:
1009 info["prefix"], info["name"] = self._posix_split_name(info["name"])
1010
1011 return self._create_header(info, USTAR_FORMAT)
1012
1013 def create_gnu_header(self, info):
1014 """Return the object as a GNU header block sequence.
1015 """
1016 info["magic"] = GNU_MAGIC
1017
1018 buf = ""
1019 if len(info["linkname"]) > LENGTH_LINK:
1020 buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK)
1021
1022 if len(info["name"]) > LENGTH_NAME:
1023 buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME)
1024
1025 return buf + self._create_header(info, GNU_FORMAT)
1026
1027 def create_pax_header(self, info, encoding, errors):
1028 """Return the object as a ustar header block. If it cannot be
1029 represented this way, prepend a pax extended header sequence
1030 with supplement information.
1031 """
1032 info["magic"] = POSIX_MAGIC
1033 pax_headers = self.pax_headers.copy()
1034
1035 # Test string fields for values that exceed the field length or cannot
1036 # be represented in ASCII encoding.
1037 for name, hname, length in (
1038 ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK),
1039 ("uname", "uname", 32), ("gname", "gname", 32)):
1040
1041 if hname in pax_headers:
1042 # The pax header has priority.
1043 continue
1044
1045 val = info[name].decode(encoding, errors)
1046
1047 # Try to encode the string as ASCII.
1048 try:
1049 val.encode("ascii")
1050 except UnicodeEncodeError:
1051 pax_headers[hname] = val
1052 continue
1053
1054 if len(info[name]) > length:
1055 pax_headers[hname] = val
1056
1057 # Test number fields for values that exceed the field limit or values
1058 # that like to be stored as float.
1059 for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)):
1060 if name in pax_headers:
1061 # The pax header has priority. Avoid overflow.
1062 info[name] = 0
1063 continue
1064
1065 val = info[name]
1066 if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float):
1067 pax_headers[name] = unicode(val)
1068 info[name] = 0
1069
1070 # Create a pax extended header if necessary.
1071 if pax_headers:
1072 buf = self._create_pax_generic_header(pax_headers)
1073 else:
1074 buf = ""
1075
1076 return buf + self._create_header(info, USTAR_FORMAT)
1077
1078 @classmethod
1079 def create_pax_global_header(cls, pax_headers):
1080 """Return the object as a pax global header block sequence.
1081 """
1082 return cls._create_pax_generic_header(pax_headers, type=XGLTYPE)
1083
1084 def _posix_split_name(self, name):
1085 """Split a name longer than 100 chars into a prefix
1086 and a name part.
1087 """
1088 prefix = name[:LENGTH_PREFIX + 1]
1089 while prefix and prefix[-1] != "/":
1090 prefix = prefix[:-1]
1091
1092 name = name[len(prefix):]
1093 prefix = prefix[:-1]
1094
1095 if not prefix or len(name) > LENGTH_NAME:
1096 raise ValueError("name is too long")
1097 return prefix, name
1098
1099 @staticmethod
1100 def _create_header(info, format):
1101 """Return a header block. info is a dictionary with file
1102 information, format must be one of the *_FORMAT constants.
1103 """
1104 parts = [
1105 stn(info.get("name", ""), 100),
1106 itn(info.get("mode", 0) & 07777, 8, format),
1107 itn(info.get("uid", 0), 8, format),
1108 itn(info.get("gid", 0), 8, format),
1109 itn(info.get("size", 0), 12, format),
1110 itn(info.get("mtime", 0), 12, format),
1111 " ", # checksum field
1112 info.get("type", REGTYPE),
1113 stn(info.get("linkname", ""), 100),
1114 stn(info.get("magic", POSIX_MAGIC), 8),
1115 stn(info.get("uname", ""), 32),
1116 stn(info.get("gname", ""), 32),
1117 itn(info.get("devmajor", 0), 8, format),
1118 itn(info.get("devminor", 0), 8, format),
1119 stn(info.get("prefix", ""), 155)
1120 ]
1121
1122 buf = struct.pack("%ds" % BLOCKSIZE, "".join(parts))
1123 chksum = calc_chksums(buf[-BLOCKSIZE:])[0]
1124 buf = buf[:-364] + "%06o\0" % chksum + buf[-357:]
1125 return buf
1126
1127 @staticmethod
1128 def _create_payload(payload):
1129 """Return the string payload filled with zero bytes
1130 up to the next 512 byte border.
1131 """
1132 blocks, remainder = divmod(len(payload), BLOCKSIZE)
1133 if remainder > 0:
1134 payload += (BLOCKSIZE - remainder) * NUL
1135 return payload
1136
1137 @classmethod
1138 def _create_gnu_long_header(cls, name, type):
1139 """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence
1140 for name.
1141 """
1142 name += NUL
1143
1144 info = {}
1145 info["name"] = "././@LongLink"
1146 info["type"] = type
1147 info["size"] = len(name)
1148 info["magic"] = GNU_MAGIC
1149
1150 # create extended header + name blocks.
1151 return cls._create_header(info, USTAR_FORMAT) + \
1152 cls._create_payload(name)
1153
1154 @classmethod
1155 def _create_pax_generic_header(cls, pax_headers, type=XHDTYPE):
1156 """Return a POSIX.1-2001 extended or global header sequence
1157 that contains a list of keyword, value pairs. The values
1158 must be unicode objects.
1159 """
1160 records = []
1161 for keyword, value in pax_headers.iteritems():
1162 keyword = keyword.encode("utf8")
1163 value = value.encode("utf8")
1164 l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n'
1165 n = p = 0
1166 while True:
1167 n = l + len(str(p))
1168 if n == p:
1169 break
1170 p = n
1171 records.append("%d %s=%s\n" % (p, keyword, value))
1172 records = "".join(records)
1173
1174 # We use a hardcoded "././@PaxHeader" name like star does
1175 # instead of the one that POSIX recommends.
1176 info = {}
1177 info["name"] = "././@PaxHeader"
1178 info["type"] = type
1179 info["size"] = len(records)
1180 info["magic"] = POSIX_MAGIC
1181
1182 # Create pax header + record blocks.
1183 return cls._create_header(info, USTAR_FORMAT) + \
1184 cls._create_payload(records)
1185
1186 @classmethod
1187 def frombuf(cls, buf):
1188 """Construct a TarInfo object from a 512 byte string buffer.
1189 """
1190 if len(buf) == 0:
1191 raise EmptyHeaderError("empty header")
1192 if len(buf) != BLOCKSIZE:
1193 raise TruncatedHeaderError("truncated header")
1194 if buf.count(NUL) == BLOCKSIZE:
1195 raise EOFHeaderError("end of file header")
1196
1197 chksum = nti(buf[148:156])
1198 if chksum not in calc_chksums(buf):
1199 raise InvalidHeaderError("bad checksum")
1200
1201 obj = cls()
1202 obj.buf = buf
1203 obj.name = nts(buf[0:100])
1204 obj.mode = nti(buf[100:108])
1205 obj.uid = nti(buf[108:116])
1206 obj.gid = nti(buf[116:124])
1207 obj.size = nti(buf[124:136])
1208 obj.mtime = nti(buf[136:148])
1209 obj.chksum = chksum
1210 obj.type = buf[156:157]
1211 obj.linkname = nts(buf[157:257])
1212 obj.uname = nts(buf[265:297])
1213 obj.gname = nts(buf[297:329])
1214 obj.devmajor = nti(buf[329:337])
1215 obj.devminor = nti(buf[337:345])
1216 prefix = nts(buf[345:500])
1217
1218 # Old V7 tar format represents a directory as a regular
1219 # file with a trailing slash.
1220 if obj.type == AREGTYPE and obj.name.endswith("/"):
1221 obj.type = DIRTYPE
1222
1223 # Remove redundant slashes from directories.
1224 if obj.isdir():
1225 obj.name = obj.name.rstrip("/")
1226
1227 # Reconstruct a ustar longname.
1228 if prefix and obj.type not in GNU_TYPES:
1229 obj.name = prefix + "/" + obj.name
1230 return obj
1231
1232 @classmethod
1233 def fromtarfile(cls, tarfile):
1234 """Return the next TarInfo object from TarFile object
1235 tarfile.
1236 """
1237 buf = tarfile.fileobj.read(BLOCKSIZE)
1238 obj = cls.frombuf(buf)
1239 obj.offset = tarfile.fileobj.tell() - BLOCKSIZE
1240 return obj._proc_member(tarfile)
1241
1242 #--------------------------------------------------------------------------
1243 # The following are methods that are called depending on the type of a
1244 # member. The entry point is _proc_member() which can be overridden in a
1245 # subclass to add custom _proc_*() methods. A _proc_*() method MUST
1246 # implement the following
1247 # operations:
1248 # 1. Set self.offset_data to the position where the data blocks begin,
1249 # if there is data that follows.
1250 # 2. Set tarfile.offset to the position where the next member's header will
1251 # begin.
1252 # 3. Return self or another valid TarInfo object.
1253 def _proc_member(self, tarfile):
1254 """Choose the right processing method depending on
1255 the type and call it.
1256 """
1257 if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK):
1258 return self._proc_gnulong(tarfile)
1259 elif self.type == GNUTYPE_SPARSE:
1260 return self._proc_sparse(tarfile)
1261 elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE):
1262 return self._proc_pax(tarfile)
1263 else:
1264 return self._proc_builtin(tarfile)
1265
1266 def _proc_builtin(self, tarfile):
1267 """Process a builtin type or an unknown type which
1268 will be treated as a regular file.
1269 """
1270 self.offset_data = tarfile.fileobj.tell()
1271 offset = self.offset_data
1272 if self.isreg() or self.type not in SUPPORTED_TYPES:
1273 # Skip the following data blocks.
1274 offset += self._block(self.size)
1275 tarfile.offset = offset
1276
1277 # Patch the TarInfo object with saved global
1278 # header information.
1279 self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors)
1280
1281 return self
1282
1283 def _proc_gnulong(self, tarfile):
1284 """Process the blocks that hold a GNU longname
1285 or longlink member.
1286 """
1287 buf = tarfile.fileobj.read(self._block(self.size))
1288
1289 # Fetch the next header and process it.
1290 try:
1291 next = self.fromtarfile(tarfile)
1292 except HeaderError:
1293 raise SubsequentHeaderError("missing or bad subsequent header")
1294
1295 # Patch the TarInfo object from the next header with
1296 # the longname information.
1297 next.offset = self.offset
1298 if self.type == GNUTYPE_LONGNAME:
1299 next.name = nts(buf)
1300 elif self.type == GNUTYPE_LONGLINK:
1301 next.linkname = nts(buf)
1302
1303 return next
1304
1305 def _proc_sparse(self, tarfile):
1306 """Process a GNU sparse header plus extra headers.
1307 """
1308 buf = self.buf
1309 sp = _ringbuffer()
1310 pos = 386
1311 lastpos = 0L
1312 realpos = 0L
1313 # There are 4 possible sparse structs in the
1314 # first header.
1315 for i in xrange(4):
1316 try:
1317 offset = nti(buf[pos:pos + 12])
1318 numbytes = nti(buf[pos + 12:pos + 24])
1319 except ValueError:
1320 break
1321 if offset > lastpos:
1322 sp.append(_hole(lastpos, offset - lastpos))
1323 sp.append(_data(offset, numbytes, realpos))
1324 realpos += numbytes
1325 lastpos = offset + numbytes
1326 pos += 24
1327
1328 isextended = ord(buf[482])
1329 origsize = nti(buf[483:495])
1330
1331 # If the isextended flag is given,
1332 # there are extra headers to process.
1333 while isextended == 1:
1334 buf = tarfile.fileobj.read(BLOCKSIZE)
1335 pos = 0
1336 for i in xrange(21):
1337 try:
1338 offset = nti(buf[pos:pos + 12])
1339 numbytes = nti(buf[pos + 12:pos + 24])
1340 except ValueError:
1341 break
1342 if offset > lastpos:
1343 sp.append(_hole(lastpos, offset - lastpos))
1344 sp.append(_data(offset, numbytes, realpos))
1345 realpos += numbytes
1346 lastpos = offset + numbytes
1347 pos += 24
1348 isextended = ord(buf[504])
1349
1350 if lastpos < origsize:
1351 sp.append(_hole(lastpos, origsize - lastpos))
1352
1353 self.sparse = sp
1354
1355 self.offset_data = tarfile.fileobj.tell()
1356 tarfile.offset = self.offset_data + self._block(self.size)
1357 self.size = origsize
1358
1359 return self
1360
1361 def _proc_pax(self, tarfile):
1362 """Process an extended or global header as described in
1363 POSIX.1-2001.
1364 """
1365 # Read the header information.
1366 buf = tarfile.fileobj.read(self._block(self.size))
1367
1368 # A pax header stores supplemental information for either
1369 # the following file (extended) or all following files
1370 # (global).
1371 if self.type == XGLTYPE:
1372 pax_headers = tarfile.pax_headers
1373 else:
1374 pax_headers = tarfile.pax_headers.copy()
1375
1376 # Parse pax header information. A record looks like that:
1377 # "%d %s=%s\n" % (length, keyword, value). length is the size
1378 # of the complete record including the length field itself and
1379 # the newline. keyword and value are both UTF-8 encoded strings.
1380 regex = re.compile(r"(\d+) ([^=]+)=", re.U)
1381 pos = 0
1382 while True:
1383 match = regex.match(buf, pos)
1384 if not match:
1385 break
1386
1387 length, keyword = match.groups()
1388 length = int(length)
1389 value = buf[match.end(2) + 1:match.start(1) + length - 1]
1390
1391 keyword = keyword.decode("utf8")
1392 value = value.decode("utf8")
1393
1394 pax_headers[keyword] = value
1395 pos += length
1396
1397 # Fetch the next header.
1398 try:
1399 next = self.fromtarfile(tarfile)
1400 except HeaderError:
1401 raise SubsequentHeaderError("missing or bad subsequent header")
1402
1403 if self.type in (XHDTYPE, SOLARIS_XHDTYPE):
1404 # Patch the TarInfo object with the extended header info.
1405 next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors)
1406 next.offset = self.offset
1407
1408 if "size" in pax_headers:
1409 # If the extended header replaces the size field,
1410 # we need to recalculate the offset where the next
1411 # header starts.
1412 offset = next.offset_data
1413 if next.isreg() or next.type not in SUPPORTED_TYPES:
1414 offset += next._block(next.size)
1415 tarfile.offset = offset
1416
1417 return next
1418
1419 def _apply_pax_info(self, pax_headers, encoding, errors):
1420 """Replace fields with supplemental information from a previous
1421 pax extended or global header.
1422 """
1423 for keyword, value in pax_headers.iteritems():
1424 if keyword not in PAX_FIELDS:
1425 continue
1426
1427 if keyword == "path":
1428 value = value.rstrip("/")
1429
1430 if keyword in PAX_NUMBER_FIELDS:
1431 try:
1432 value = PAX_NUMBER_FIELDS[keyword](value)
1433 except ValueError:
1434 value = 0
1435 else:
1436 value = uts(value, encoding, errors)
1437
1438 setattr(self, keyword, value)
1439
1440 self.pax_headers = pax_headers.copy()
1441
1442 def _block(self, count):
1443 """Round up a byte count by BLOCKSIZE and return it,
1444 e.g. _block(834) => 1024.
1445 """
1446 blocks, remainder = divmod(count, BLOCKSIZE)
1447 if remainder:
1448 blocks += 1
1449 return blocks * BLOCKSIZE
1450
1451 def isreg(self):
1452 return self.type in REGULAR_TYPES
1453 def isfile(self):
1454 return self.isreg()
1455 def isdir(self):
1456 return self.type == DIRTYPE
1457 def issym(self):
1458 return self.type == SYMTYPE
1459 def islnk(self):
1460 return self.type == LNKTYPE
1461 def ischr(self):
1462 return self.type == CHRTYPE
1463 def isblk(self):
1464 return self.type == BLKTYPE
1465 def isfifo(self):
1466 return self.type == FIFOTYPE
1467 def issparse(self):
1468 return self.type == GNUTYPE_SPARSE
1469 def isdev(self):
1470 return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE)
1471# class TarInfo
1472
1473class TarFile(object):
1474 """The TarFile Class provides an interface to tar archives.
1475 """
1476
1477 debug = 0 # May be set from 0 (no msgs) to 3 (all msgs)
1478
1479 dereference = False # If true, add content of linked file to the
1480 # tar file, else the link.
1481
1482 ignore_zeros = False # If true, skips empty or invalid blocks and
1483 # continues processing.
1484
1485 errorlevel = 1 # If 0, fatal errors only appear in debug
1486 # messages (if debug >= 0). If > 0, errors
1487 # are passed to the caller as exceptions.
1488
1489 format = DEFAULT_FORMAT # The format to use when creating an archive.
1490
1491 encoding = ENCODING # Encoding for 8-bit character strings.
1492
1493 errors = None # Error handler for unicode conversion.
1494
1495 tarinfo = TarInfo # The default TarInfo class to use.
1496
1497 fileobject = ExFileObject # The default ExFileObject class to use.
1498
1499 def __init__(self, name=None, mode="r", fileobj=None, format=None,
1500 tarinfo=None, dereference=None, ignore_zeros=None, encoding=None,
1501 errors=None, pax_headers=None, debug=None, errorlevel=None):
1502 """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to
1503 read from an existing archive, 'a' to append data to an existing
1504 file or 'w' to create a new file overwriting an existing one. `mode'
1505 defaults to 'r'.
1506 If `fileobj' is given, it is used for reading or writing data. If it
1507 can be determined, `mode' is overridden by `fileobj's mode.
1508 `fileobj' is not closed, when TarFile is closed.
1509 """
1510 if len(mode) > 1 or mode not in "raw":
1511 raise ValueError("mode must be 'r', 'a' or 'w'")
1512 self.mode = mode
1513 self._mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode]
1514
1515 if not fileobj:
1516 if self.mode == "a" and not os.path.exists(name):
1517 # Create nonexistent files in append mode.
1518 self.mode = "w"
1519 self._mode = "wb"
1520 fileobj = bltn_open(name, self._mode)
1521 self._extfileobj = False
1522 else:
1523 if name is None and hasattr(fileobj, "name"):
1524 name = fileobj.name
1525 if hasattr(fileobj, "mode"):
1526 self._mode = fileobj.mode
1527 self._extfileobj = True
1528 self.name = os.path.abspath(name) if name else None
1529 self.fileobj = fileobj
1530
1531 # Init attributes.
1532 if format is not None:
1533 self.format = format
1534 if tarinfo is not None:
1535 self.tarinfo = tarinfo
1536 if dereference is not None:
1537 self.dereference = dereference
1538 if ignore_zeros is not None:
1539 self.ignore_zeros = ignore_zeros
1540 if encoding is not None:
1541 self.encoding = encoding
1542
1543 if errors is not None:
1544 self.errors = errors
1545 elif mode == "r":
1546 self.errors = "utf-8"
1547 else:
1548 self.errors = "strict"
1549
1550 if pax_headers is not None and self.format == PAX_FORMAT:
1551 self.pax_headers = pax_headers
1552 else:
1553 self.pax_headers = {}
1554
1555 if debug is not None:
1556 self.debug = debug
1557 if errorlevel is not None:
1558 self.errorlevel = errorlevel
1559
1560 # Init datastructures.
1561 self.closed = False
1562 self.members = [] # list of members as TarInfo objects
1563 self._loaded = False # flag if all members have been read
1564 self.offset = self.fileobj.tell()
1565 # current position in the archive file
1566 self.inodes = {} # dictionary caching the inodes of
1567 # archive members already added
1568
1569 try:
1570 if self.mode == "r":
1571 self.firstmember = None
1572 self.firstmember = self.next()
1573
1574 if self.mode == "a":
1575 # Move to the end of the archive,
1576 # before the first empty block.
1577 while True:
1578 self.fileobj.seek(self.offset)
1579 try:
1580 tarinfo = self.tarinfo.fromtarfile(self)
1581 self.members.append(tarinfo)
1582 except EOFHeaderError:
1583 self.fileobj.seek(self.offset)
1584 break
1585 except HeaderError, e:
1586 raise ReadError(str(e))
1587
1588 if self.mode in "aw":
1589 self._loaded = True
1590
1591 if self.pax_headers:
1592 buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy())
1593 self.fileobj.write(buf)
1594 self.offset += len(buf)
1595 except:
1596 if not self._extfileobj:
1597 self.fileobj.close()
1598 self.closed = True
1599 raise
1600
1601 def _getposix(self):
1602 return self.format == USTAR_FORMAT
1603 def _setposix(self, value):
1604 import warnings
1605 warnings.warn("use the format attribute instead", DeprecationWarning,
1606 2)
1607 if value:
1608 self.format = USTAR_FORMAT
1609 else:
1610 self.format = GNU_FORMAT
1611 posix = property(_getposix, _setposix)
1612
1613 #--------------------------------------------------------------------------
1614 # Below are the classmethods which act as alternate constructors to the
1615 # TarFile class. The open() method is the only one that is needed for
1616 # public use; it is the "super"-constructor and is able to select an
1617 # adequate "sub"-constructor for a particular compression using the mapping
1618 # from OPEN_METH.
1619 #
1620 # This concept allows one to subclass TarFile without losing the comfort of
1621 # the super-constructor. A sub-constructor is registered and made available
1622 # by adding it to the mapping in OPEN_METH.
1623
1624 @classmethod
1625 def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs):
1626 """Open a tar archive for reading, writing or appending. Return
1627 an appropriate TarFile class.
1628
1629 mode:
1630 'r' or 'r:*' open for reading with transparent compression
1631 'r:' open for reading exclusively uncompressed
1632 'r:gz' open for reading with gzip compression
1633 'r:bz2' open for reading with bzip2 compression
1634 'a' or 'a:' open for appending, creating the file if necessary
1635 'w' or 'w:' open for writing without compression
1636 'w:gz' open for writing with gzip compression
1637 'w:bz2' open for writing with bzip2 compression
1638
1639 'r|*' open a stream of tar blocks with transparent compression
1640 'r|' open an uncompressed stream of tar blocks for reading
1641 'r|gz' open a gzip compressed stream of tar blocks
1642 'r|bz2' open a bzip2 compressed stream of tar blocks
1643 'w|' open an uncompressed stream for writing
1644 'w|gz' open a gzip compressed stream for writing
1645 'w|bz2' open a bzip2 compressed stream for writing
1646 """
1647
1648 if not name and not fileobj:
1649 raise ValueError("nothing to open")
1650
1651 if mode in ("r", "r:*"):
1652 # Find out which *open() is appropriate for opening the file.
1653 for comptype in cls.OPEN_METH:
1654 func = getattr(cls, cls.OPEN_METH[comptype])
1655 if fileobj is not None:
1656 saved_pos = fileobj.tell()
1657 try:
1658 return func(name, "r", fileobj, **kwargs)
1659 except (ReadError, CompressionError), e:
1660 if fileobj is not None:
1661 fileobj.seek(saved_pos)
1662 continue
1663 raise ReadError("file could not be opened successfully")
1664
1665 elif ":" in mode:
1666 filemode, comptype = mode.split(":", 1)
1667 filemode = filemode or "r"
1668 comptype = comptype or "tar"
1669
1670 # Select the *open() function according to
1671 # given compression.
1672 if comptype in cls.OPEN_METH:
1673 func = getattr(cls, cls.OPEN_METH[comptype])
1674 else:
1675 raise CompressionError("unknown compression type %r" % comptype)
1676 return func(name, filemode, fileobj, **kwargs)
1677
1678 elif "|" in mode:
1679 filemode, comptype = mode.split("|", 1)
1680 filemode = filemode or "r"
1681 comptype = comptype or "tar"
1682
1683 if filemode not in "rw":
1684 raise ValueError("mode must be 'r' or 'w'")
1685
1686 t = cls(name, filemode,
1687 _Stream(name, filemode, comptype, fileobj, bufsize),
1688 **kwargs)
1689 t._extfileobj = False
1690 return t
1691
1692 elif mode in "aw":
1693 return cls.taropen(name, mode, fileobj, **kwargs)
1694
1695 raise ValueError("undiscernible mode")
1696
1697 @classmethod
1698 def taropen(cls, name, mode="r", fileobj=None, **kwargs):
1699 """Open uncompressed tar archive name for reading or writing.
1700 """
1701 if len(mode) > 1 or mode not in "raw":
1702 raise ValueError("mode must be 'r', 'a' or 'w'")
1703 return cls(name, mode, fileobj, **kwargs)
1704
1705 @classmethod
1706 def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
1707 """Open gzip compressed tar archive name for reading or writing.
1708 Appending is not allowed.
1709 """
1710 if len(mode) > 1 or mode not in "rw":
1711 raise ValueError("mode must be 'r' or 'w'")
1712
1713 try:
1714 import gzip
1715 gzip.GzipFile
1716 except (ImportError, AttributeError):
1717 raise CompressionError("gzip module is not available")
1718
1719 if fileobj is None:
1720 fileobj = bltn_open(name, mode + "b")
1721
1722 try:
1723 t = cls.taropen(name, mode,
1724 gzip.GzipFile(name, mode, compresslevel, fileobj),
1725 **kwargs)
1726 except IOError:
1727 raise ReadError("not a gzip file")
1728 t._extfileobj = False
1729 return t
1730
1731 @classmethod
1732 def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
1733 """Open bzip2 compressed tar archive name for reading or writing.
1734 Appending is not allowed.
1735 """
1736 if len(mode) > 1 or mode not in "rw":
1737 raise ValueError("mode must be 'r' or 'w'.")
1738
1739 try:
1740 import bz2
1741 except ImportError:
1742 raise CompressionError("bz2 module is not available")
1743
1744 if fileobj is not None:
1745 fileobj = _BZ2Proxy(fileobj, mode)
1746 else:
1747 fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
1748
1749 try:
1750 t = cls.taropen(name, mode, fileobj, **kwargs)
1751 except (IOError, EOFError):
1752 raise ReadError("not a bzip2 file")
1753 t._extfileobj = False
1754 return t
1755
1756 # All *open() methods are registered here.
1757 OPEN_METH = {
1758 "tar": "taropen", # uncompressed tar
1759 "gz": "gzopen", # gzip compressed tar
1760 "bz2": "bz2open" # bzip2 compressed tar
1761 }
1762
1763 #--------------------------------------------------------------------------
1764 # The public methods which TarFile provides:
1765
1766 def close(self):
1767 """Close the TarFile. In write-mode, two finishing zero blocks are
1768 appended to the archive.
1769 """
1770 if self.closed:
1771 return
1772
1773 if self.mode in "aw":
1774 self.fileobj.write(NUL * (BLOCKSIZE * 2))
1775 self.offset += (BLOCKSIZE * 2)
1776 # fill up the end with zero-blocks
1777 # (like option -b20 for tar does)
1778 blocks, remainder = divmod(self.offset, RECORDSIZE)
1779 if remainder > 0:
1780 self.fileobj.write(NUL * (RECORDSIZE - remainder))
1781
1782 if not self._extfileobj:
1783 self.fileobj.close()
1784 self.closed = True
1785
1786 def getmember(self, name):
1787 """Return a TarInfo object for member `name'. If `name' can not be
1788 found in the archive, KeyError is raised. If a member occurs more
1789 than once in the archive, its last occurrence is assumed to be the
1790 most up-to-date version.
1791 """
1792 tarinfo = self._getmember(name)
1793 if tarinfo is None:
1794 raise KeyError("filename %r not found" % name)
1795 return tarinfo
1796
1797 def getmembers(self):
1798 """Return the members of the archive as a list of TarInfo objects. The
1799 list has the same order as the members in the archive.
1800 """
1801 self._check()
1802 if not self._loaded: # if we want to obtain a list of
1803 self._load() # all members, we first have to
1804 # scan the whole archive.
1805 return self.members
1806
1807 def getnames(self):
1808 """Return the members of the archive as a list of their names. It has
1809 the same order as the list returned by getmembers().
1810 """
1811 return [tarinfo.name for tarinfo in self.getmembers()]
1812
1813 def gettarinfo(self, name=None, arcname=None, fileobj=None):
1814 """Create a TarInfo object for either the file `name' or the file
1815 object `fileobj' (using os.fstat on its file descriptor). You can
1816 modify some of the TarInfo's attributes before you add it using
1817 addfile(). If given, `arcname' specifies an alternative name for the
1818 file in the archive.
1819 """
1820 self._check("aw")
1821
1822 # When fileobj is given, replace name by
1823 # fileobj's real name.
1824 if fileobj is not None:
1825 name = fileobj.name
1826
1827 # Building the name of the member in the archive.
1828 # Backward slashes are converted to forward slashes,
1829 # Absolute paths are turned to relative paths.
1830 if arcname is None:
1831 arcname = name
1832 drv, arcname = os.path.splitdrive(arcname)
1833 arcname = arcname.replace(os.sep, "/")
1834 arcname = arcname.lstrip("/")
1835
1836 # Now, fill the TarInfo object with
1837 # information specific for the file.
1838 tarinfo = self.tarinfo()
1839 tarinfo.tarfile = self
1840
1841 # Use os.stat or os.lstat, depending on platform
1842 # and if symlinks shall be resolved.
1843 if fileobj is None:
1844 if hasattr(os, "lstat") and not self.dereference:
1845 statres = os.lstat(name)
1846 else:
1847 statres = os.stat(name)
1848 else:
1849 statres = os.fstat(fileobj.fileno())
1850 linkname = ""
1851
1852 stmd = statres.st_mode
1853 if stat.S_ISREG(stmd):
1854 inode = (statres.st_ino, statres.st_dev)
1855 if not self.dereference and statres.st_nlink > 1 and \
1856 inode in self.inodes and arcname != self.inodes[inode]:
1857 # Is it a hardlink to an already
1858 # archived file?
1859 type = LNKTYPE
1860 linkname = self.inodes[inode]
1861 else:
1862 # The inode is added only if its valid.
1863 # For win32 it is always 0.
1864 type = REGTYPE
1865 if inode[0]:
1866 self.inodes[inode] = arcname
1867 elif stat.S_ISDIR(stmd):
1868 type = DIRTYPE
1869 elif stat.S_ISFIFO(stmd):
1870 type = FIFOTYPE
1871 elif stat.S_ISLNK(stmd):
1872 type = SYMTYPE
1873 linkname = os.readlink(name)
1874 elif stat.S_ISCHR(stmd):
1875 type = CHRTYPE
1876 elif stat.S_ISBLK(stmd):
1877 type = BLKTYPE
1878 else:
1879 return None
1880
1881 # Fill the TarInfo object with all
1882 # information we can get.
1883 tarinfo.name = arcname
1884 tarinfo.mode = stmd
1885 tarinfo.uid = statres.st_uid
1886 tarinfo.gid = statres.st_gid
1887 if type == REGTYPE:
1888 tarinfo.size = statres.st_size
1889 else:
1890 tarinfo.size = 0L
1891 tarinfo.mtime = statres.st_mtime
1892 tarinfo.type = type
1893 tarinfo.linkname = linkname
1894 if pwd:
1895 try:
1896 tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
1897 except KeyError:
1898 pass
1899 if grp:
1900 try:
1901 tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
1902 except KeyError:
1903 pass
1904
1905 if type in (CHRTYPE, BLKTYPE):
1906 if hasattr(os, "major") and hasattr(os, "minor"):
1907 tarinfo.devmajor = os.major(statres.st_rdev)
1908 tarinfo.devminor = os.minor(statres.st_rdev)
1909 return tarinfo
1910
1911 def list(self, verbose=True):
1912 """Print a table of contents to sys.stdout. If `verbose' is False, only
1913 the names of the members are printed. If it is True, an `ls -l'-like
1914 output is produced.
1915 """
1916 self._check()
1917
1918 for tarinfo in self:
1919 if verbose:
1920 print filemode(tarinfo.mode),
1921 print "%s/%s" % (tarinfo.uname or tarinfo.uid,
1922 tarinfo.gname or tarinfo.gid),
1923 if tarinfo.ischr() or tarinfo.isblk():
1924 print "%10s" % ("%d,%d" \
1925 % (tarinfo.devmajor, tarinfo.devminor)),
1926 else:
1927 print "%10d" % tarinfo.size,
1928 print "%d-%02d-%02d %02d:%02d:%02d" \
1929 % time.localtime(tarinfo.mtime)[:6],
1930
1931 print tarinfo.name + ("/" if tarinfo.isdir() else ""),
1932
1933 if verbose:
1934 if tarinfo.issym():
1935 print "->", tarinfo.linkname,
1936 if tarinfo.islnk():
1937 print "link to", tarinfo.linkname,
1938 print
1939
1940 def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
1941 """Add the file `name' to the archive. `name' may be any type of file
1942 (directory, fifo, symbolic link, etc.). If given, `arcname'
1943 specifies an alternative name for the file in the archive.
1944 Directories are added recursively by default. This can be avoided by
1945 setting `recursive' to False. `exclude' is a function that should
1946 return True for each filename to be excluded. `filter' is a function
1947 that expects a TarInfo object argument and returns the changed
1948 TarInfo object, if it returns None the TarInfo object will be
1949 excluded from the archive.
1950 """
1951 self._check("aw")
1952
1953 if arcname is None:
1954 arcname = name
1955
1956 # Exclude pathnames.
1957 if exclude is not None:
1958 import warnings
1959 warnings.warn("use the filter argument instead",
1960 DeprecationWarning, 2)
1961 if exclude(name):
1962 self._dbg(2, "tarfile: Excluded %r" % name)
1963 return
1964
1965 # Skip if somebody tries to archive the archive...
1966 if self.name is not None and os.path.abspath(name) == self.name:
1967 self._dbg(2, "tarfile: Skipped %r" % name)
1968 return
1969
1970 self._dbg(1, name)
1971
1972 # Create a TarInfo object from the file.
1973 tarinfo = self.gettarinfo(name, arcname)
1974
1975 if tarinfo is None:
1976 self._dbg(1, "tarfile: Unsupported type %r" % name)
1977 return
1978
1979 # Change or exclude the TarInfo object.
1980 if filter is not None:
1981 tarinfo = filter(tarinfo)
1982 if tarinfo is None:
1983 self._dbg(2, "tarfile: Excluded %r" % name)
1984 return
1985
1986 # Append the tar header and data to the archive.
1987 if tarinfo.isreg():
1988 f = bltn_open(name, "rb")
1989 self.addfile(tarinfo, f)
1990 f.close()
1991
1992 elif tarinfo.isdir():
1993 self.addfile(tarinfo)
1994 if recursive:
1995 for f in os.listdir(name):
1996 self.add(os.path.join(name, f), os.path.join(arcname, f),
1997 recursive, exclude, filter)
1998
1999 else:
2000 self.addfile(tarinfo)
2001
2002 def addfile(self, tarinfo, fileobj=None):
2003 """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is
2004 given, tarinfo.size bytes are read from it and added to the archive.
2005 You can create TarInfo objects using gettarinfo().
2006 On Windows platforms, `fileobj' should always be opened with mode
2007 'rb' to avoid irritation about the file size.
2008 """
2009 self._check("aw")
2010
2011 tarinfo = copy.copy(tarinfo)
2012
2013 buf = tarinfo.tobuf(self.format, self.encoding, self.errors)
2014 self.fileobj.write(buf)
2015 self.offset += len(buf)
2016
2017 # If there's data to follow, append it.
2018 if fileobj is not None:
2019 copyfileobj(fileobj, self.fileobj, tarinfo.size)
2020 blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
2021 if remainder > 0:
2022 self.fileobj.write(NUL * (BLOCKSIZE - remainder))
2023 blocks += 1
2024 self.offset += blocks * BLOCKSIZE
2025
2026 self.members.append(tarinfo)
2027
2028 def extractall(self, path=".", members=None):
2029 """Extract all members from the archive to the current working
2030 directory and set owner, modification time and permissions on
2031 directories afterwards. `path' specifies a different directory
2032 to extract to. `members' is optional and must be a subset of the
2033 list returned by getmembers().
2034 """
2035 directories = []
2036
2037 if members is None:
2038 members = self
2039
2040 for tarinfo in members:
2041 if tarinfo.isdir():
2042 # Extract directories with a safe mode.
2043 directories.append(tarinfo)
2044 tarinfo = copy.copy(tarinfo)
2045 tarinfo.mode = 0700
2046 self.extract(tarinfo, path)
2047
2048 # Reverse sort directories.
2049 directories.sort(key=operator.attrgetter('name'))
2050 directories.reverse()
2051
2052 # Set correct owner, mtime and filemode on directories.
2053 for tarinfo in directories:
2054 dirpath = os.path.join(path, tarinfo.name)
2055 try:
2056 self.chown(tarinfo, dirpath)
2057 self.utime(tarinfo, dirpath)
2058 self.chmod(tarinfo, dirpath)
2059 except ExtractError, e:
2060 if self.errorlevel > 1:
2061 raise
2062 else:
2063 self._dbg(1, "tarfile: %s" % e)
2064
2065 def extract(self, member, path=""):
2066 """Extract a member from the archive to the current working directory,
2067 using its full name. Its file information is extracted as accurately
2068 as possible. `member' may be a filename or a TarInfo object. You can
2069 specify a different directory using `path'.
2070 """
2071 self._check("r")
2072
2073 if isinstance(member, basestring):
2074 tarinfo = self.getmember(member)
2075 else:
2076 tarinfo = member
2077
2078 # Prepare the link target for makelink().
2079 if tarinfo.islnk():
2080 tarinfo._link_target = os.path.join(path, tarinfo.linkname)
2081
2082 try:
2083 self._extract_member(tarinfo, os.path.join(path, tarinfo.name))
2084 except EnvironmentError, e:
2085 if self.errorlevel > 0:
2086 raise
2087 else:
2088 if e.filename is None:
2089 self._dbg(1, "tarfile: %s" % e.strerror)
2090 else:
2091 self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
2092 except ExtractError, e:
2093 if self.errorlevel > 1:
2094 raise
2095 else:
2096 self._dbg(1, "tarfile: %s" % e)
2097
2098 def extractfile(self, member):
2099 """Extract a member from the archive as a file object. `member' may be
2100 a filename or a TarInfo object. If `member' is a regular file, a
2101 file-like object is returned. If `member' is a link, a file-like
2102 object is constructed from the link's target. If `member' is none of
2103 the above, None is returned.
2104 The file-like object is read-only and provides the following
2105 methods: read(), readline(), readlines(), seek() and tell()
2106 """
2107 self._check("r")
2108
2109 if isinstance(member, basestring):
2110 tarinfo = self.getmember(member)
2111 else:
2112 tarinfo = member
2113
2114 if tarinfo.isreg():
2115 return self.fileobject(self, tarinfo)
2116
2117 elif tarinfo.type not in SUPPORTED_TYPES:
2118 # If a member's type is unknown, it is treated as a
2119 # regular file.
2120 return self.fileobject(self, tarinfo)
2121
2122 elif tarinfo.islnk() or tarinfo.issym():
2123 if isinstance(self.fileobj, _Stream):
2124 # A small but ugly workaround for the case that someone tries
2125 # to extract a (sym)link as a file-object from a non-seekable
2126 # stream of tar blocks.
2127 raise StreamError("cannot extract (sym)link as file object")
2128 else:
2129 # A (sym)link's file object is its target's file object.
2130 return self.extractfile(self._find_link_target(tarinfo))
2131 else:
2132 # If there's no data associated with the member (directory, chrdev,
2133 # blkdev, etc.), return None instead of a file object.
2134 return None
2135
2136 def _extract_member(self, tarinfo, targetpath):
2137 """Extract the TarInfo object tarinfo to a physical
2138 file called targetpath.
2139 """
2140 # Fetch the TarInfo object for the given name
2141 # and build the destination pathname, replacing
2142 # forward slashes to platform specific separators.
2143 targetpath = targetpath.rstrip("/")
2144 targetpath = targetpath.replace("/", os.sep)
2145
2146 # Create all upper directories.
2147 upperdirs = os.path.dirname(targetpath)
2148 if upperdirs and not os.path.exists(upperdirs):
2149 # Create directories that are not part of the archive with
2150 # default permissions.
2151 os.makedirs(upperdirs)
2152
2153 if tarinfo.islnk() or tarinfo.issym():
2154 self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname))
2155 else:
2156 self._dbg(1, tarinfo.name)
2157
2158 if tarinfo.isreg():
2159 self.makefile(tarinfo, targetpath)
2160 elif tarinfo.isdir():
2161 self.makedir(tarinfo, targetpath)
2162 elif tarinfo.isfifo():
2163 self.makefifo(tarinfo, targetpath)
2164 elif tarinfo.ischr() or tarinfo.isblk():
2165 self.makedev(tarinfo, targetpath)
2166 elif tarinfo.islnk() or tarinfo.issym():
2167 self.makelink(tarinfo, targetpath)
2168 elif tarinfo.type not in SUPPORTED_TYPES:
2169 self.makeunknown(tarinfo, targetpath)
2170 else:
2171 self.makefile(tarinfo, targetpath)
2172
2173 self.chown(tarinfo, targetpath)
2174 if not tarinfo.issym():
2175 self.chmod(tarinfo, targetpath)
2176 self.utime(tarinfo, targetpath)
2177
2178 #--------------------------------------------------------------------------
2179 # Below are the different file methods. They are called via
2180 # _extract_member() when extract() is called. They can be replaced in a
2181 # subclass to implement other functionality.
2182
2183 def makedir(self, tarinfo, targetpath):
2184 """Make a directory called targetpath.
2185 """
2186 try:
2187 # Use a safe mode for the directory, the real mode is set
2188 # later in _extract_member().
2189 os.mkdir(targetpath, 0700)
2190 except EnvironmentError, e:
2191 if e.errno != errno.EEXIST:
2192 raise
2193
2194 def makefile(self, tarinfo, targetpath):
2195 """Make a file called targetpath.
2196 """
2197 source = self.extractfile(tarinfo)
2198 target = bltn_open(targetpath, "wb")
2199 copyfileobj(source, target)
2200 source.close()
2201 target.close()
2202
2203 def makeunknown(self, tarinfo, targetpath):
2204 """Make a file from a TarInfo object with an unknown type
2205 at targetpath.
2206 """
2207 self.makefile(tarinfo, targetpath)
2208 self._dbg(1, "tarfile: Unknown file type %r, " \
2209 "extracted as regular file." % tarinfo.type)
2210
2211 def makefifo(self, tarinfo, targetpath):
2212 """Make a fifo called targetpath.
2213 """
2214 if hasattr(os, "mkfifo"):
2215 os.mkfifo(targetpath)
2216 else:
2217 raise ExtractError("fifo not supported by system")
2218
2219 def makedev(self, tarinfo, targetpath):
2220 """Make a character or block device called targetpath.
2221 """
2222 if not hasattr(os, "mknod") or not hasattr(os, "makedev"):
2223 raise ExtractError("special devices not supported by system")
2224
2225 mode = tarinfo.mode
2226 if tarinfo.isblk():
2227 mode |= stat.S_IFBLK
2228 else:
2229 mode |= stat.S_IFCHR
2230
2231 os.mknod(targetpath, mode,
2232 os.makedev(tarinfo.devmajor, tarinfo.devminor))
2233
2234 def makelink(self, tarinfo, targetpath):
2235 """Make a (symbolic) link called targetpath. If it cannot be created
2236 (platform limitation), we try to make a copy of the referenced file
2237 instead of a link.
2238 """
2239 if hasattr(os, "symlink") and hasattr(os, "link"):
2240 # For systems that support symbolic and hard links.
2241 if tarinfo.issym():
2242 if os.path.lexists(targetpath):
2243 os.unlink(targetpath)
2244 os.symlink(tarinfo.linkname, targetpath)
2245 else:
2246 # See extract().
2247 if os.path.exists(tarinfo._link_target):
2248 if os.path.lexists(targetpath):
2249 os.unlink(targetpath)
2250 os.link(tarinfo._link_target, targetpath)
2251 else:
2252 self._extract_member(self._find_link_target(tarinfo), targetpath)
2253 else:
2254 try:
2255 self._extract_member(self._find_link_target(tarinfo), targetpath)
2256 except KeyError:
2257 raise ExtractError("unable to resolve link inside archive")
2258
2259 def chown(self, tarinfo, targetpath):
2260 """Set owner of targetpath according to tarinfo.
2261 """
2262 if pwd and hasattr(os, "geteuid") and os.geteuid() == 0:
2263 # We have to be root to do so.
2264 try:
2265 g = grp.getgrnam(tarinfo.gname)[2]
2266 except KeyError:
2267 try:
2268 g = grp.getgrgid(tarinfo.gid)[2]
2269 except KeyError:
2270 g = os.getgid()
2271 try:
2272 u = pwd.getpwnam(tarinfo.uname)[2]
2273 except KeyError:
2274 try:
2275 u = pwd.getpwuid(tarinfo.uid)[2]
2276 except KeyError:
2277 u = os.getuid()
2278 try:
2279 if tarinfo.issym() and hasattr(os, "lchown"):
2280 os.lchown(targetpath, u, g)
2281 else:
2282 if sys.platform != "os2emx":
2283 os.chown(targetpath, u, g)
2284 except EnvironmentError, e:
2285 raise ExtractError("could not change owner")
2286
2287 def chmod(self, tarinfo, targetpath):
2288 """Set file permissions of targetpath according to tarinfo.
2289 """
2290 if hasattr(os, 'chmod'):
2291 try:
2292 os.chmod(targetpath, tarinfo.mode)
2293 except EnvironmentError, e:
2294 raise ExtractError("could not change mode")
2295
2296 def utime(self, tarinfo, targetpath):
2297 """Set modification time of targetpath according to tarinfo.
2298 """
2299 if not hasattr(os, 'utime'):
2300 return
2301 try:
2302 os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
2303 except EnvironmentError, e:
2304 raise ExtractError("could not change modification time")
2305
2306 #--------------------------------------------------------------------------
2307 def next(self):
2308 """Return the next member of the archive as a TarInfo object, when
2309 TarFile is opened for reading. Return None if there is no more
2310 available.
2311 """
2312 self._check("ra")
2313 if self.firstmember is not None:
2314 m = self.firstmember
2315 self.firstmember = None
2316 return m
2317
2318 # Read the next block.
2319 self.fileobj.seek(self.offset)
2320 tarinfo = None
2321 while True:
2322 try:
2323 tarinfo = self.tarinfo.fromtarfile(self)
2324 except EOFHeaderError, e:
2325 if self.ignore_zeros:
2326 self._dbg(2, "0x%X: %s" % (self.offset, e))
2327 self.offset += BLOCKSIZE
2328 continue
2329 except InvalidHeaderError, e:
2330 if self.ignore_zeros:
2331 self._dbg(2, "0x%X: %s" % (self.offset, e))
2332 self.offset += BLOCKSIZE
2333 continue
2334 elif self.offset == 0:
2335 raise ReadError(str(e))
2336 except EmptyHeaderError:
2337 if self.offset == 0:
2338 raise ReadError("empty file")
2339 except TruncatedHeaderError, e:
2340 if self.offset == 0:
2341 raise ReadError(str(e))
2342 except SubsequentHeaderError, e:
2343 raise ReadError(str(e))
2344 break
2345
2346 if tarinfo is not None:
2347 self.members.append(tarinfo)
2348 else:
2349 self._loaded = True
2350
2351 return tarinfo
2352
2353 #--------------------------------------------------------------------------
2354 # Little helper methods:
2355
2356 def _getmember(self, name, tarinfo=None, normalize=False):
2357 """Find an archive member by name from bottom to top.
2358 If tarinfo is given, it is used as the starting point.
2359 """
2360 # Ensure that all members have been loaded.
2361 members = self.getmembers()
2362
2363 # Limit the member search list up to tarinfo.
2364 if tarinfo is not None:
2365 members = members[:members.index(tarinfo)]
2366
2367 if normalize:
2368 name = os.path.normpath(name)
2369
2370 for member in reversed(members):
2371 if normalize:
2372 member_name = os.path.normpath(member.name)
2373 else:
2374 member_name = member.name
2375
2376 if name == member_name:
2377 return member
2378
2379 def _load(self):
2380 """Read through the entire archive file and look for readable
2381 members.
2382 """
2383 while True:
2384 tarinfo = self.next()
2385 if tarinfo is None:
2386 break
2387 self._loaded = True
2388
2389 def _check(self, mode=None):
2390 """Check if TarFile is still open, and if the operation's mode
2391 corresponds to TarFile's mode.
2392 """
2393 if self.closed:
2394 raise IOError("%s is closed" % self.__class__.__name__)
2395 if mode is not None and self.mode not in mode:
2396 raise IOError("bad operation for mode %r" % self.mode)
2397
2398 def _find_link_target(self, tarinfo):
2399 """Find the target member of a symlink or hardlink member in the
2400 archive.
2401 """
2402 if tarinfo.issym():
2403 # Always search the entire archive.
2404 linkname = os.path.dirname(tarinfo.name) + "/" + tarinfo.linkname
2405 limit = None
2406 else:
2407 # Search the archive before the link, because a hard link is
2408 # just a reference to an already archived file.
2409 linkname = tarinfo.linkname
2410 limit = tarinfo
2411
2412 member = self._getmember(linkname, tarinfo=limit, normalize=True)
2413 if member is None:
2414 raise KeyError("linkname %r not found" % linkname)
2415 return member
2416
2417 def __iter__(self):
2418 """Provide an iterator object.
2419 """
2420 if self._loaded:
2421 return iter(self.members)
2422 else:
2423 return TarIter(self)
2424
2425 def _dbg(self, level, msg):
2426 """Write debugging output to sys.stderr.
2427 """
2428 if level <= self.debug:
2429 print >> sys.stderr, msg
2430
2431 def __enter__(self):
2432 self._check()
2433 return self
2434
2435 def __exit__(self, type, value, traceback):
2436 if type is None:
2437 self.close()
2438 else:
2439 # An exception occurred. We must not call close() because
2440 # it would try to write end-of-archive blocks and padding.
2441 if not self._extfileobj:
2442 self.fileobj.close()
2443 self.closed = True
2444# class TarFile
2445
2446class TarIter:
2447 """Iterator Class.
2448
2449 for tarinfo in TarFile(...):
2450 suite...
2451 """
2452
2453 def __init__(self, tarfile):
2454 """Construct a TarIter object.
2455 """
2456 self.tarfile = tarfile
2457 self.index = 0
2458 def __iter__(self):
2459 """Return iterator object.
2460 """
2461 return self
2462 def next(self):
2463 """Return the next item using TarFile's next() method.
2464 When all members have been read, set TarFile as _loaded.
2465 """
2466 # Fix for SF #1100429: Under rare circumstances it can
2467 # happen that getmembers() is called during iteration,
2468 # which will cause TarIter to stop prematurely.
2469 if not self.tarfile._loaded:
2470 tarinfo = self.tarfile.next()
2471 if not tarinfo:
2472 self.tarfile._loaded = True
2473 raise StopIteration
2474 else:
2475 try:
2476 tarinfo = self.tarfile.members[self.index]
2477 except IndexError:
2478 raise StopIteration
2479 self.index += 1
2480 return tarinfo
2481
2482# Helper classes for sparse file support
2483class _section:
2484 """Base class for _data and _hole.
2485 """
2486 def __init__(self, offset, size):
2487 self.offset = offset
2488 self.size = size
2489 def __contains__(self, offset):
2490 return self.offset <= offset < self.offset + self.size
2491
2492class _data(_section):
2493 """Represent a data section in a sparse file.
2494 """
2495 def __init__(self, offset, size, realpos):
2496 _section.__init__(self, offset, size)
2497 self.realpos = realpos
2498
2499class _hole(_section):
2500 """Represent a hole section in a sparse file.
2501 """
2502 pass
2503
2504class _ringbuffer(list):
2505 """Ringbuffer class which increases performance
2506 over a regular list.
2507 """
2508 def __init__(self):
2509 self.idx = 0
2510 def find(self, offset):
2511 idx = self.idx
2512 while True:
2513 item = self[idx]
2514 if offset in item:
2515 break
2516 idx += 1
2517 if idx == len(self):
2518 idx = 0
2519 if idx == self.idx:
2520 # End of File
2521 return None
2522 self.idx = idx
2523 return item
2524
2525#---------------------------------------------
2526# zipfile compatible TarFile class
2527#---------------------------------------------
2528TAR_PLAIN = 0 # zipfile.ZIP_STORED
2529TAR_GZIPPED = 8 # zipfile.ZIP_DEFLATED
2530class TarFileCompat:
2531 """TarFile class compatible with standard module zipfile's
2532 ZipFile class.
2533 """
2534 def __init__(self, file, mode="r", compression=TAR_PLAIN):
2535 from warnings import warnpy3k
2536 warnpy3k("the TarFileCompat class has been removed in Python 3.0",
2537 stacklevel=2)
2538 if compression == TAR_PLAIN:
2539 self.tarfile = TarFile.taropen(file, mode)
2540 elif compression == TAR_GZIPPED:
2541 self.tarfile = TarFile.gzopen(file, mode)
2542 else:
2543 raise ValueError("unknown compression constant")
2544 if mode[0:1] == "r":
2545 members = self.tarfile.getmembers()
2546 for m in members:
2547 m.filename = m.name
2548 m.file_size = m.size
2549 m.date_time = time.gmtime(m.mtime)[:6]
2550 def namelist(self):
2551 return map(lambda m: m.name, self.infolist())
2552 def infolist(self):
2553 return filter(lambda m: m.type in REGULAR_TYPES,
2554 self.tarfile.getmembers())
2555 def printdir(self):
2556 self.tarfile.list()
2557 def testzip(self):
2558 return
2559 def getinfo(self, name):
2560 return self.tarfile.getmember(name)
2561 def read(self, name):
2562 return self.tarfile.extractfile(self.tarfile.getmember(name)).read()
2563 def write(self, filename, arcname=None, compress_type=None):
2564 self.tarfile.add(filename, arcname)
2565 def writestr(self, zinfo, bytes):
2566 try:
2567 from cStringIO import StringIO
2568 except ImportError:
2569 from StringIO import StringIO
2570 import calendar
2571 tinfo = TarInfo(zinfo.filename)
2572 tinfo.size = len(bytes)
2573 tinfo.mtime = calendar.timegm(zinfo.date_time)
2574 self.tarfile.addfile(tinfo, StringIO(bytes))
2575 def close(self):
2576 self.tarfile.close()
2577#class TarFileCompat
2578
2579#--------------------
2580# exported functions
2581#--------------------
2582def is_tarfile(name):
2583 """Return True if name points to a tar archive that we
2584 are able to handle, else return False.
2585 """
2586 try:
2587 t = open(name)
2588 t.close()
2589 return True
2590 except TarError:
2591 return False
2592
2593bltn_open = open
2594open = TarFile.open