(wip) S3 upload

app/document.py

@@ -33,11 +33,32 @@
 document = Blueprint('document', __name__)
 
 
+
 class UploadForm(Form):
+    """
+    Upload form using normal upload
+    """
+
     file = FileField('The file to review')
     title = TextField('Title',
                       description='The title of your document (may be blank)')
 
+    def save_validated_form(self):
+        return documents.save(self.file.data)
+
+
+class S3UploadForm(Form):
+    """
+    Upload directly to S3
+    """
+
+
+def upload_form():
+    if current_app.config.get('UPLOAD_TYPE') == 'S3':
+        return S3UploadForm()
+    else:
+        return UploadForm()
+
 
 @document.route('/upload', methods=['POST'])
 def upload():
@@ -46,10 +67,10 @@ def upload():
 
     :query revises: ID this doc is a new revision of.
     """
-    form = UploadForm()
+    form = upload_form()
     if form.validate_on_submit():
         try:
-            filename = documents.save(form.file.data)
+            filename = form.save_validated_form()
         except UploadNotAllowed:
             flash('Unsupported file type')
             return redirect(url_for('bp.home'))
@@ -78,7 +99,7 @@ def view(id):
     id = kore_id(id)
     doc = Document.query.get_or_404(id)
     form_comm = CommentForm(docid=id)
-    form_up = UploadForm()
+    form_up = upload_form()
     form_share = ShareForm()
     comments = Comment.query.filter_by(doc=id)
     readOnly = not current_user.is_authenticated()

app/factory.py

@@ -23,6 +23,7 @@
 from comment import comment
 from document import document
 from key import get_secret_key
+from s3 import s3
 from uploads import documents
 from vendor.slack_log_handler import SlackLogHandler
 from views import bp
@@ -207,6 +208,7 @@ def register_blueprints(app):
         annotation,
         auth,
         audioann,
+        s3,
         ]
     for blueprint in blueprints:
         app.register_blueprint(blueprint)

app/s3.py

@@ -0,0 +1,44 @@
+import hmac
+import json
+import sha
+from base64 import b64encode
+from datetime import datetime
+from datetime import timedelta
+from uuid import uuid4
+
+from flask import Blueprint
+from flask import current_app
+from flask import jsonify
+
+s3 = Blueprint('s3', __name__)
+
+
+def make_policy():
+    bucket = current_app.config.get('S3_BUCKET')
+    assert bucket is not None, "S3 upload is not configured"
+    now = datetime.now()
+    delta = timedelta(hours=1)
+    expiration = (now + delta).strftime('%Y-%m-%dT%H:%M:%S.000Z')
+    conditions = [{'bucket': bucket},
+                  {'acl': 'public-read'},
+                  ["starts-with", "$key", "uploads/"],
+                  {'success_action_status': '201'},
+                  ]
+    policy = {'expiration': expiration,
+              'conditions': conditions,
+              }
+    return b64encode(json.dumps(policy).replace('\n', '').replace('\r', ''))
+
+
+def sign_policy(policy):
+    key = current_app.config.get('AWS_SECRET_ACCESS_KEY')
+    assert key is not None, "S3 upload is not configured"
+    return b64encode(hmac.new(key, policy, sha).digest())
+
+
+@s3.route('/s3_sign')
+def sign():
+    key = "uploads/" + uuid4().hex
+    policy = make_policy()
+    signature = sign_policy(policy)
+    return jsonify(key=key, policy=policy, signature=signature)

app/views.py

@@ -10,7 +10,7 @@
 from flask.ext.wtf import Form
 from wtforms import TextField
 
-from .document import UploadForm
+from .document import upload_form
 from .models import Annotation
 from .models import Document
 from .tools import kore_id
@@ -23,7 +23,7 @@ def home():
     """
     Home page.
     """
-    kwargs = {'form': UploadForm()}
+    kwargs = {'form': upload_form()}
     if current_user.is_authenticated():
         documents = Document.mine()
         kwargs['documents'] = documents

static/coffee/lib/s3uploader.coffee

@@ -0,0 +1,35 @@
+init = ->
+
+class S3Uploader
+  constructor: (params) ->
+    @paramsUrl = params.paramsUrl
+    @$status = params.$status
+    @$form = params.$form
+    @$uploadBtn = params.$uploadBtn
+
+  log: (status) ->
+    @$status.html status
+
+  start: ->
+    @$upload_button.click ->
+        @$form.find("input[type=file]").click()
+    @$form.fileupload
+        autoUpload: true
+        dataType: "xml"
+        add: (event, data) ->
+            log "fetching params"
+            $.get(@paramsUrl).done (params) =>
+                @$form.find('input[name=key]').val(params.key)
+                @$form.find('input[name=policy]').val(params.policy)
+                @$form.find('input[name=signature]').val(params.signature)
+                data.submit()
+        send: (event, data) ->
+            log "sending"
+        progress: (event, data) ->
+            @$progress_bar.css "width", "#{Math.round((event.loaded / event.total) * 1000) / 10}%"
+        fail: (event, data) ->
+            log "failure"
+        success: (event, data) ->
+            log "success"
+        done: (event, data) ->
+            log "done"

static/coffee/pages/upload.coffee

@@ -0,0 +1,7 @@
+upload_init = ->
+  s3u = new S3Uploader
+    paramsUrl: '/params'
+    $status: $('#status')
+    $form: $("#upload_form")
+    $uploadBtn: $("#upload_button")
+  s3u.start()

tests/test_s3.py

@@ -0,0 +1,45 @@
+import hmac
+import json
+import sha
+from base64 import b64decode
+from base64 import b64encode
+from datetime import datetime
+from datetime import timedelta
+from time import strptime
+
+from flask import url_for
+
+from common import RankoTestCase
+
+
+class S3TestCase(RankoTestCase):
+
+    def setUp(self):
+        self.app.config['S3_BUCKET'] = 'my-awesome-bucket'
+        self.app.config['AWS_SECRET_ACCESS_KEY'] = 'my-awesome-key'
+
+    def test_sign(self):
+        r = self.client.get(url_for('s3.sign'))
+        params = r.json
+
+        self.assertTrue(params['key'].startswith('uploads/'))
+
+        encoded_policy = params['policy']
+        policy = json.loads(b64decode(encoded_policy))
+        now = datetime.now()
+        expiration = policy['expiration']
+        expiration_time = strptime(expiration, '%Y-%m-%dT%H:%M:%S.000Z')
+        expiration_datetime = datetime(*expiration_time[:6])
+        delta = expiration_datetime - now
+        self.assertLess(delta, timedelta(hours=1))
+
+        conditions = policy['conditions']
+        self.assertIn({'acl': 'public-read'}, conditions)
+        self.assertIn(["starts-with", "$key", "uploads/"], conditions)
+        self.assertIn({'bucket': 'my-awesome-bucket'}, conditions)
+        self.assertIn({'success_action_status': '201'}, conditions)
+
+        signature = params['signature']
+        key = 'my-awesome-key'
+        expected = b64encode(hmac.new(key, encoded_policy, sha).digest())
+        self.assertEqual(signature, expected)