diff --git a/app/document.py b/app/document.py index 4c49de4..bc8da8b 100644 --- a/app/document.py +++ b/app/document.py @@ -33,11 +33,32 @@ document = Blueprint('document', __name__) + class UploadForm(Form): + """ + Upload form using normal upload + """ + file = FileField('The file to review') title = TextField('Title', description='The title of your document (may be blank)') + def save_validated_form(self): + return documents.save(self.file.data) + + +class S3UploadForm(Form): + """ + Upload directly to S3 + """ + + +def upload_form(): + if current_app.config.get('UPLOAD_TYPE') == 'S3': + return S3UploadForm() + else: + return UploadForm() + @document.route('/upload', methods=['POST']) def upload(): @@ -46,10 +67,10 @@ def upload(): :query revises: ID this doc is a new revision of. """ - form = UploadForm() + form = upload_form() if form.validate_on_submit(): try: - filename = documents.save(form.file.data) + filename = form.save_validated_form() except UploadNotAllowed: flash('Unsupported file type') return redirect(url_for('bp.home')) @@ -78,7 +99,7 @@ def view(id): id = kore_id(id) doc = Document.query.get_or_404(id) form_comm = CommentForm(docid=id) - form_up = UploadForm() + form_up = upload_form() form_share = ShareForm() comments = Comment.query.filter_by(doc=id) readOnly = not current_user.is_authenticated() diff --git a/app/factory.py b/app/factory.py index 6f76fe2..a30d96b 100644 --- a/app/factory.py +++ b/app/factory.py @@ -23,6 +23,7 @@ from comment import comment from document import document from key import get_secret_key +from s3 import s3 from uploads import documents from vendor.slack_log_handler import SlackLogHandler from views import bp @@ -207,6 +208,7 @@ def register_blueprints(app): annotation, auth, audioann, + s3, ] for blueprint in blueprints: app.register_blueprint(blueprint) diff --git a/app/s3.py b/app/s3.py new file mode 100644 index 0000000..3d30d43 --- /dev/null +++ b/app/s3.py @@ -0,0 +1,44 @@ +import hmac +import json +import sha +from base64 import b64encode +from datetime import datetime +from datetime import timedelta +from uuid import uuid4 + +from flask import Blueprint +from flask import current_app +from flask import jsonify + +s3 = Blueprint('s3', __name__) + + +def make_policy(): + bucket = current_app.config.get('S3_BUCKET') + assert bucket is not None, "S3 upload is not configured" + now = datetime.now() + delta = timedelta(hours=1) + expiration = (now + delta).strftime('%Y-%m-%dT%H:%M:%S.000Z') + conditions = [{'bucket': bucket}, + {'acl': 'public-read'}, + ["starts-with", "$key", "uploads/"], + {'success_action_status': '201'}, + ] + policy = {'expiration': expiration, + 'conditions': conditions, + } + return b64encode(json.dumps(policy).replace('\n', '').replace('\r', '')) + + +def sign_policy(policy): + key = current_app.config.get('AWS_SECRET_ACCESS_KEY') + assert key is not None, "S3 upload is not configured" + return b64encode(hmac.new(key, policy, sha).digest()) + + +@s3.route('/s3_sign') +def sign(): + key = "uploads/" + uuid4().hex + policy = make_policy() + signature = sign_policy(policy) + return jsonify(key=key, policy=policy, signature=signature) diff --git a/app/views.py b/app/views.py index e1a556f..71f2acf 100644 --- a/app/views.py +++ b/app/views.py @@ -10,7 +10,7 @@ from flask.ext.wtf import Form from wtforms import TextField -from .document import UploadForm +from .document import upload_form from .models import Annotation from .models import Document from .tools import kore_id @@ -23,7 +23,7 @@ def home(): """ Home page. """ - kwargs = {'form': UploadForm()} + kwargs = {'form': upload_form()} if current_user.is_authenticated(): documents = Document.mine() kwargs['documents'] = documents diff --git a/static/coffee/lib/s3uploader.coffee b/static/coffee/lib/s3uploader.coffee new file mode 100644 index 0000000..6c51ca1 --- /dev/null +++ b/static/coffee/lib/s3uploader.coffee @@ -0,0 +1,35 @@ +init = -> + +class S3Uploader + constructor: (params) -> + @paramsUrl = params.paramsUrl + @$status = params.$status + @$form = params.$form + @$uploadBtn = params.$uploadBtn + + log: (status) -> + @$status.html status + + start: -> + @$upload_button.click -> + @$form.find("input[type=file]").click() + @$form.fileupload + autoUpload: true + dataType: "xml" + add: (event, data) -> + log "fetching params" + $.get(@paramsUrl).done (params) => + @$form.find('input[name=key]').val(params.key) + @$form.find('input[name=policy]').val(params.policy) + @$form.find('input[name=signature]').val(params.signature) + data.submit() + send: (event, data) -> + log "sending" + progress: (event, data) -> + @$progress_bar.css "width", "#{Math.round((event.loaded / event.total) * 1000) / 10}%" + fail: (event, data) -> + log "failure" + success: (event, data) -> + log "success" + done: (event, data) -> + log "done" diff --git a/static/coffee/pages/upload.coffee b/static/coffee/pages/upload.coffee new file mode 100644 index 0000000..5ca405a --- /dev/null +++ b/static/coffee/pages/upload.coffee @@ -0,0 +1,7 @@ +upload_init = -> + s3u = new S3Uploader + paramsUrl: '/params' + $status: $('#status') + $form: $("#upload_form") + $uploadBtn: $("#upload_button") + s3u.start() diff --git a/tests/test_s3.py b/tests/test_s3.py new file mode 100644 index 0000000..4d63104 --- /dev/null +++ b/tests/test_s3.py @@ -0,0 +1,45 @@ +import hmac +import json +import sha +from base64 import b64decode +from base64 import b64encode +from datetime import datetime +from datetime import timedelta +from time import strptime + +from flask import url_for + +from common import RankoTestCase + + +class S3TestCase(RankoTestCase): + + def setUp(self): + self.app.config['S3_BUCKET'] = 'my-awesome-bucket' + self.app.config['AWS_SECRET_ACCESS_KEY'] = 'my-awesome-key' + + def test_sign(self): + r = self.client.get(url_for('s3.sign')) + params = r.json + + self.assertTrue(params['key'].startswith('uploads/')) + + encoded_policy = params['policy'] + policy = json.loads(b64decode(encoded_policy)) + now = datetime.now() + expiration = policy['expiration'] + expiration_time = strptime(expiration, '%Y-%m-%dT%H:%M:%S.000Z') + expiration_datetime = datetime(*expiration_time[:6]) + delta = expiration_datetime - now + self.assertLess(delta, timedelta(hours=1)) + + conditions = policy['conditions'] + self.assertIn({'acl': 'public-read'}, conditions) + self.assertIn(["starts-with", "$key", "uploads/"], conditions) + self.assertIn({'bucket': 'my-awesome-bucket'}, conditions) + self.assertIn({'success_action_status': '201'}, conditions) + + signature = params['signature'] + key = 'my-awesome-key' + expected = b64encode(hmac.new(key, encoded_policy, sha).digest()) + self.assertEqual(signature, expected)