From 7104b2107db592756f4090ba772869eb86d6a44f Mon Sep 17 00:00:00 2001 From: jfriedli Date: Fri, 27 Mar 2020 10:59:16 -0700 Subject: file removal background job --- test/test.py | 53 +++++++++++++++++++++++++++++++++++++ test/test_api.py | 49 ++++++++++++++++++++++++++++++++-- test/test_file_removal_scheduler.py | 48 +++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 test/test_file_removal_scheduler.py (limited to 'test') diff --git a/test/test.py b/test/test.py index 34245d9..a7e6627 100644 --- a/test/test.py +++ b/test/test.py @@ -1,9 +1,12 @@ +import base64 import unittest import tempfile import shutil import io import os +from unittest.mock import patch + import main @@ -62,6 +65,33 @@ class Mat2WebTestCase(unittest.TestCase): rv.data) self.assertEqual(rv.status_code, 200) + def test_get_upload_no_selected_file(self): + rv = self.app.post('/', + data=dict( + file=(io.BytesIO(b""), ''), + ), follow_redirects=True) + self.assertIn(b'No selected file', + rv.data) + self.assertEqual(rv.status_code, 200) + + def test_failed_cleaning(self): + zip_file_bytes = base64.b64decode( + 'UEsDBBQACAAIAPicPE8AAAAAAAAAAAAAAAAXACAAZmFpbGluZy5ub3Qtd29ya2luZy1le' + 'HRVVA0AB+Saj13kmo9d5JqPXXV4CwABBOkDAAAE6QMAAAMAUEsHCAAAAAACAAAAAAAAAFBL' + 'AwQUAAgACAD6nDxPAAAAAAAAAAAAAAAACQAgAHRlc3QuanNvblVUDQAH6JqPXeiaj13omo9d' + 'dXgLAAEE6QMAAATpAwAAAwBQSwcIAAAAAAIAAAAAAAAAUEsBAhQDFAAIAAgA+Jw8TwAAAAACA' + 'AAAAAAAABcAIAAAAAAAAAAAAKSBAAAAAGZhaWxpbmcubm90LXdvcmtpbmctZXh0VVQNAAfkmo9' + 'd5JqPXeSaj111eAsAAQTpAwAABOkDAABQSwECFAMUAAgACAD6nDxPAAAAAAIAAAAAAAAACQAgA' + 'AAAAAAAAAAApIFnAAAAdGVzdC5qc29uVVQNAAfomo9d6JqPXeiaj111eAsAAQTpAwAABOkDAAB' + 'QSwUGAAAAAAIAAgC8AAAAwAAAAAAA' + ) + rv = self.app.post('/', + data=dict( + file=(io.BytesIO(zip_file_bytes), 'test.zip'), + ), follow_redirects=True) + self.assertIn(b'Unable to clean',rv.data) + self.assertEqual(rv.status_code, 200) + def test_get_upload_no_file_name(self): rv = self.app.post('/', data=dict( @@ -97,6 +127,29 @@ class Mat2WebTestCase(unittest.TestCase): rv = self.app.get('/download/70623619c449a040968cdbea85945bf384fa30ed2d5d24fa3/test.cleaned.txt') self.assertEqual(rv.status_code, 302) + @patch('file_removal_scheduler.random.randint') + def test_upload_leftover(self, randint_mock): + randint_mock.return_value = 0 + os.environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = '0' + app = main.create_app() + self.upload_folder = tempfile.mkdtemp() + app.config.update( + TESTING=True, + UPLOAD_FOLDER=self.upload_folder + ) + app = app.test_client() + + request = self.app.post('/', + data=dict( + file=(io.BytesIO(b"Some text"), 'test.txt'), + ), follow_redirects=True) + self.assertEqual(request.status_code, 200) + request = app.get( + b'/download/4c2e9e6da31a64c70623619c449a040968cdbea85945bf384fa30ed2d5d24fa3/test.cleaned.txt' + ) + self.assertEqual(302, request.status_code) + os.environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = '9999' + if __name__ == '__main__': unittest.main() diff --git a/test/test_api.py b/test/test_api.py index de297c4..3074bd5 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -4,9 +4,10 @@ import json import os import shutil import zipfile - from six import BytesIO +from unittest.mock import patch + import main @@ -122,6 +123,23 @@ class Mat2APITestCase(unittest.TestCase): rv = self.app.get('/api/extension', headers={'Origin': 'origin1.gnu'}) self.assertEqual(rv.headers['Access-Control-Allow-Origin'], 'origin1.gnu') + def test_api_cleaning_failed(self): + request = self.app.post('/api/upload', + data='{"file_name": "test_name.zip", ' + '"file": "UEsDBBQACAAIAPicPE8AAAAAAAAAAAAAAAAXACAAZmFpbGluZy5ub3Qt' + 'd29ya2luZy1leHRVVA0AB+Saj13kmo9d5JqPXXV4CwABBOkDAAAE6QMAAAMAUEsHCAAA' + 'AAACAAAAAAAAAFBLAwQUAAgACAD6nDxPAAAAAAAAAAAAAAAACQAgAHRlc3QuanNvblVUD' + 'QAH6JqPXeiaj13omo9ddXgLAAEE6QMAAATpAwAAAwBQSwcIAAAAAAIAAAAAAAAAUEsBAhQD' + 'FAAIAAgA+Jw8TwAAAAACAAAAAAAAABcAIAAAAAAAAAAAAKSBAAAAAGZhaWxpbmcubm90LXd' + 'vcmtpbmctZXh0VVQNAAfkmo9d5JqPXeSaj111eAsAAQTpAwAABOkDAABQSwECFAMUAAgACAD6' + 'nDxPAAAAAAIAAAAAAAAACQAgAAAAAAAAAAAApIFnAAAAdGVzdC5qc29uVVQNAAfomo9d6JqPXe' + 'iaj111eAsAAQTpAwAABOkDAABQSwUGAAAAAAIAAgC8AAAAwAAAAAAA"}', + headers={'content-type': 'application/json'} + ) + error = json.loads(request.data.decode('utf-8'))['message'] + self.assertEqual(error, 'Unable to clean application/zip') + + def test_api_download(self): request = self.app.post('/api/upload', data='{"file_name": "test_name.jpg", ' @@ -263,7 +281,6 @@ class Mat2APITestCase(unittest.TestCase): ) response = json.loads(request.data.decode('utf-8')) - print(response) self.assertEqual(response['message']['download_list'][0]['0'][0]['file_name'][0], 'required field') self.assertEqual(response['message']['download_list'][0]['0'][0]['key'][0], 'required field') self.assertEqual(request.status_code, 400) @@ -344,6 +361,34 @@ class Mat2APITestCase(unittest.TestCase): response = json.loads(request.data.decode('utf-8')) self.assertEqual('File not found', response['message']) + @patch('file_removal_scheduler.random.randint') + def test_api_upload_leftover(self, randint_mock): + os.environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = '0' + app = main.create_app() + self.upload_folder = tempfile.mkdtemp() + app.config.update( + TESTING=True, + UPLOAD_FOLDER=self.upload_folder + ) + app = app.test_client() + randint_mock.return_value = 1 + self.upload_download_test_jpg_and_assert_response_code(app, 200) + randint_mock.return_value = 0 + self.upload_download_test_jpg_and_assert_response_code(app, 404) + + os.environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = '9999' + + def upload_download_test_jpg_and_assert_response_code(self, app, code): + request = app.post('/api/upload', + data='{"file_name": "test_name.jpg", ' + '"file": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAf' + 'FcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="}', + headers={'content-type': 'application/json'} + ) + download_link = json.loads(request.data.decode('utf-8'))['download_link'] + request = app.get(download_link) + self.assertEqual(code, request.status_code) + if __name__ == '__main__': unittest.main() diff --git a/test/test_file_removal_scheduler.py b/test/test_file_removal_scheduler.py new file mode 100644 index 0000000..7f6771a --- /dev/null +++ b/test/test_file_removal_scheduler.py @@ -0,0 +1,48 @@ +import unittest +import tempfile +from os import path, environ +import shutil + +import file_removal_scheduler +import main + + +class Mat2WebTestCase(unittest.TestCase): + def setUp(self): + self.upload_folder = tempfile.mkdtemp() + app = main.create_app() + app.config.update( + TESTING=True, + UPLOAD_FOLDER=self.upload_folder + ) + self.app = app + + def test_removal(self): + filename = 'test_name.cleaned.jpg' + environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = '0' + open(path.join(self.upload_folder, filename), 'a').close() + self.assertTrue(path.exists(path.join(self.upload_folder, ))) + for i in range(0, 11): + file_removal_scheduler.run_file_removal_job(self.app.config['UPLOAD_FOLDER']) + self.assertFalse(path.exists(path.join(self.upload_folder, filename))) + + open(path.join(self.upload_folder, filename), 'a').close() + file_removal_scheduler.run_file_removal_job(self.app.config['UPLOAD_FOLDER']) + self.assertTrue(path.exists(path.join(self.upload_folder, ))) + + def test_non_removal(self): + filename = u'i_should_no_be_removed.txt' + environ['MAT2_MAX_FILE_AGE_FOR_REMOVAL'] = '9999999' + open(path.join(self.upload_folder, filename), 'a').close() + self.assertTrue(path.exists(path.join(self.upload_folder, filename))) + for i in range(0, 11): + file_removal_scheduler.run_file_removal_job(self.app.config['UPLOAD_FOLDER']) + self.assertTrue(path.exists(path.join(self.upload_folder, filename))) + + def tearDown(self): + shutil.rmtree(self.upload_folder) + + +if __name__ == '__main__': + unittest.main() + -- cgit v1.3