diff options
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | matweb/file_removal_scheduler.py | 4 | ||||
| -rw-r--r-- | matweb/rest_api.py | 2 | ||||
| -rw-r--r-- | matweb/utils.py | 6 | ||||
| -rw-r--r-- | test/test.py | 3 | ||||
| -rw-r--r-- | test/test_api.py | 5 | ||||
| -rw-r--r-- | test/test_file_removal_scheduler.py | 1 |
7 files changed, 18 insertions, 5 deletions
| @@ -110,6 +110,7 @@ The `file` parameter is the base64 encoded file which will be cleaned. | |||
| 110 | **Example Response:** | 110 | **Example Response:** |
| 111 | ```json | 111 | ```json |
| 112 | { | 112 | { |
| 113 | "inactive_after_sec": 120, | ||
| 113 | "output_filename": "fancy.cleaned.jpg", | 114 | "output_filename": "fancy.cleaned.jpg", |
| 114 | "mime": "image/jpg", | 115 | "mime": "image/jpg", |
| 115 | "key": "81a541f9ebc0233d419d25ed39908b16f82be26a783f32d56c381559e84e6161", | 116 | "key": "81a541f9ebc0233d419d25ed39908b16f82be26a783f32d56c381559e84e6161", |
| @@ -175,6 +176,7 @@ The `key` parameter is the key from a previously uploaded file. | |||
| 175 | **Example Response:** | 176 | **Example Response:** |
| 176 | ```json | 177 | ```json |
| 177 | { | 178 | { |
| 179 | "inactive_after_sec": 120, | ||
| 178 | "output_filename": "files.2cd225d5-2d75-44a2-9f26-e120a87e4279.cleaned.zip", | 180 | "output_filename": "files.2cd225d5-2d75-44a2-9f26-e120a87e4279.cleaned.zip", |
| 179 | "mime": "application/zip", | 181 | "mime": "application/zip", |
| 180 | "key": "5ee4cf8821226340d3d5ed16bd2e1b435234a9ad218f282b489a85d116e7a4c4", | 182 | "key": "5ee4cf8821226340d3d5ed16bd2e1b435234a9ad218f282b489a85d116e7a4c4", |
diff --git a/matweb/file_removal_scheduler.py b/matweb/file_removal_scheduler.py index 2ce7912..59bb9ca 100644 --- a/matweb/file_removal_scheduler.py +++ b/matweb/file_removal_scheduler.py | |||
| @@ -4,6 +4,8 @@ import sys | |||
| 4 | import os | 4 | import os |
| 5 | import random | 5 | import random |
| 6 | 6 | ||
| 7 | from matweb import utils | ||
| 8 | |||
| 7 | 9 | ||
| 8 | def run_file_removal_job(upload_folder_path): | 10 | def run_file_removal_job(upload_folder_path): |
| 9 | if random.randint(0, 10) == 0: | 11 | if random.randint(0, 10) == 0: |
| @@ -18,7 +20,7 @@ def delete_file_when_too_old(filepath): | |||
| 18 | last_time = time.time() - file_mod_time | 20 | last_time = time.time() - file_mod_time |
| 19 | 21 | ||
| 20 | # if file is older than our configured max timeframe, delete it | 22 | # if file is older than our configured max timeframe, delete it |
| 21 | if last_time > int(os.environ.get('MAT2_MAX_FILE_AGE_FOR_REMOVAL', 15 * 60)): | 23 | if last_time > utils.get_file_removal_max_age_sec(): |
| 22 | try: | 24 | try: |
| 23 | os.remove(filepath) | 25 | os.remove(filepath) |
| 24 | except OSError: | 26 | except OSError: |
diff --git a/matweb/rest_api.py b/matweb/rest_api.py index a07d2d2..f893fca 100644 --- a/matweb/rest_api.py +++ b/matweb/rest_api.py | |||
| @@ -48,6 +48,7 @@ class APIUpload(Resource): | |||
| 48 | 48 | ||
| 49 | key, secret, meta_after, output_filename = utils.cleanup(parser, filepath, self.upload_folder) | 49 | key, secret, meta_after, output_filename = utils.cleanup(parser, filepath, self.upload_folder) |
| 50 | return utils.return_file_created_response( | 50 | return utils.return_file_created_response( |
| 51 | utils.get_file_removal_max_age_sec(), | ||
| 51 | output_filename, | 52 | output_filename, |
| 52 | mime, | 53 | mime, |
| 53 | key, | 54 | key, |
| @@ -133,6 +134,7 @@ class APIBulkDownloadCreator(Resource): | |||
| 133 | abort(500, message='Unable to clean %s' % mime) | 134 | abort(500, message='Unable to clean %s' % mime) |
| 134 | key, secret, meta_after, output_filename = utils.cleanup(parser, zip_path, self.upload_folder) | 135 | key, secret, meta_after, output_filename = utils.cleanup(parser, zip_path, self.upload_folder) |
| 135 | return { | 136 | return { |
| 137 | 'inactive_after_sec': utils.get_file_removal_max_age_sec(), | ||
| 136 | 'output_filename': output_filename, | 138 | 'output_filename': output_filename, |
| 137 | 'mime': mime, | 139 | 'mime': mime, |
| 138 | 'key': key, | 140 | 'key': key, |
diff --git a/matweb/utils.py b/matweb/utils.py index 20c213d..915d735 100644 --- a/matweb/utils.py +++ b/matweb/utils.py | |||
| @@ -35,6 +35,7 @@ def check_upload_folder(upload_folder): | |||
| 35 | 35 | ||
| 36 | 36 | ||
| 37 | def return_file_created_response( | 37 | def return_file_created_response( |
| 38 | inactive_after_sec: int, | ||
| 38 | output_filename: str, | 39 | output_filename: str, |
| 39 | mime: str, | 40 | mime: str, |
| 40 | key: str, | 41 | key: str, |
| @@ -44,6 +45,7 @@ def return_file_created_response( | |||
| 44 | download_link: str | 45 | download_link: str |
| 45 | ) -> dict: | 46 | ) -> dict: |
| 46 | return { | 47 | return { |
| 48 | 'inactive_after_sec': inactive_after_sec, | ||
| 47 | 'output_filename': output_filename, | 49 | 'output_filename': output_filename, |
| 48 | 'mime': mime, | 50 | 'mime': mime, |
| 49 | 'key': key, | 51 | 'key': key, |
| @@ -106,3 +108,7 @@ def is_valid_api_download_file(filename: str, key: str, secret: str, upload_fold | |||
| 106 | if hmac.compare_digest(hash_file(complete_path, secret), key) is False: | 108 | if hmac.compare_digest(hash_file(complete_path, secret), key) is False: |
| 107 | abort(400, message='The file hash does not match') | 109 | abort(400, message='The file hash does not match') |
| 108 | return complete_path, filepath | 110 | return complete_path, filepath |
| 111 | |||
| 112 | |||
| 113 | def get_file_removal_max_age_sec() -> int: | ||
| 114 | return int(os.environ.get('MAT2_MAX_FILE_AGE_FOR_REMOVAL', 15 * 60)) | ||
diff --git a/test/test.py b/test/test.py index 6734dcf..97e7567 100644 --- a/test/test.py +++ b/test/test.py | |||
| @@ -170,7 +170,7 @@ class Mat2WebTestCase(TestCase): | |||
| 170 | 170 | ||
| 171 | request = app.get(self.get_context_variable('download_uri')) | 171 | request = app.get(self.get_context_variable('download_uri')) |
| 172 | self.assertEqual(302, request.status_code) | 172 | self.assertEqual(302, request.status_code) |
| 173 | os.environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = '9999' | 173 | os.environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = str(15*60) |
| 174 | 174 | ||
| 175 | def test_info_page(self): | 175 | def test_info_page(self): |
| 176 | rv = self.client.get('/info') | 176 | rv = self.client.get('/info') |
| @@ -191,7 +191,6 @@ class Mat2WebTestCase(TestCase): | |||
| 191 | self.assertIn(b'Invalid Filename', rv.data) | 191 | self.assertIn(b'Invalid Filename', rv.data) |
| 192 | 192 | ||
| 193 | 193 | ||
| 194 | |||
| 195 | if __name__ == '__main__': | 194 | if __name__ == '__main__': |
| 196 | unittest.main() | 195 | unittest.main() |
| 197 | 196 | ||
diff --git a/test/test_api.py b/test/test_api.py index 63955d0..1a50aa6 100644 --- a/test/test_api.py +++ b/test/test_api.py | |||
| @@ -42,7 +42,7 @@ class Mat2APITestCase(unittest.TestCase): | |||
| 42 | self.assertEqual(request.status_code, 200) | 42 | self.assertEqual(request.status_code, 200) |
| 43 | 43 | ||
| 44 | data = request.get_json() | 44 | data = request.get_json() |
| 45 | self.assertEqual(data['output_filename'], 'test_name.cleaned.jpg') | 45 | self.assertEqual(data['inactive_after_sec'], 15 * 60) |
| 46 | self.assertEqual(data['output_filename'], 'test_name.cleaned.jpg') | 46 | self.assertEqual(data['output_filename'], 'test_name.cleaned.jpg') |
| 47 | self.assertEqual(data['mime'], 'image/jpeg') | 47 | self.assertEqual(data['mime'], 'image/jpeg') |
| 48 | self.assertEqual(len(data['secret']), 64) | 48 | self.assertEqual(len(data['secret']), 64) |
| @@ -237,6 +237,7 @@ class Mat2APITestCase(unittest.TestCase): | |||
| 237 | 237 | ||
| 238 | self.assertIn('files.', response['output_filename']) | 238 | self.assertIn('files.', response['output_filename']) |
| 239 | self.assertIn('cleaned.zip', response['output_filename']) | 239 | self.assertIn('cleaned.zip', response['output_filename']) |
| 240 | self.assertEquals(15 * 60, response['inactive_after_sec']) | ||
| 240 | self.assertIn(response['mime'], 'application/zip') | 241 | self.assertIn(response['mime'], 'application/zip') |
| 241 | self.assertEqual(response['meta_after'], {}) | 242 | self.assertEqual(response['meta_after'], {}) |
| 242 | 243 | ||
| @@ -387,7 +388,7 @@ class Mat2APITestCase(unittest.TestCase): | |||
| 387 | randint_mock.return_value = 0 | 388 | randint_mock.return_value = 0 |
| 388 | self.upload_download_test_jpg_and_assert_response_code(app, 404) | 389 | self.upload_download_test_jpg_and_assert_response_code(app, 404) |
| 389 | 390 | ||
| 390 | os.environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = '9999' | 391 | os.environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = str(15 * 60) |
| 391 | 392 | ||
| 392 | def upload_download_test_jpg_and_assert_response_code(self, app, code): | 393 | def upload_download_test_jpg_and_assert_response_code(self, app, code): |
| 393 | request = app.post('/api/upload', | 394 | request = app.post('/api/upload', |
diff --git a/test/test_file_removal_scheduler.py b/test/test_file_removal_scheduler.py index 1210cb2..563fc73 100644 --- a/test/test_file_removal_scheduler.py +++ b/test/test_file_removal_scheduler.py | |||
| @@ -42,6 +42,7 @@ class Mat2WebTestCase(unittest.TestCase): | |||
| 42 | randint_mock.return_value = 0 | 42 | randint_mock.return_value = 0 |
| 43 | file_removal_scheduler.run_file_removal_job(self.app.config['UPLOAD_FOLDER']) | 43 | file_removal_scheduler.run_file_removal_job(self.app.config['UPLOAD_FOLDER']) |
| 44 | self.assertTrue(path.exists(path.join(self.upload_folder, filename))) | 44 | self.assertTrue(path.exists(path.join(self.upload_folder, filename))) |
| 45 | environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = str(15 * 60) | ||
| 45 | 46 | ||
| 46 | def tearDown(self): | 47 | def tearDown(self): |
| 47 | shutil.rmtree(self.upload_folder) | 48 | shutil.rmtree(self.upload_folder) |
