[ENG-11731] Real-time Download Dashboard

tests/core/test_remote_logging.py

@@ -3,6 +3,213 @@
 from waterbutler.core import remote_logging
 
 
+class TestLogToCallback:
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize('completed, expected', [(False, False), (True, True)])
+    async def test_download_action_sets_completed_flag(self, monkeypatch, completed, expected):
+        captured = {}
+
+        async def fake_send_signed_request(method, url, payload):
+            captured['payload'] = payload
+            return 200, b'success'
+
+        monkeypatch.setattr(remote_logging.utils, 'send_signed_request', fake_send_signed_request)
+
+        class DummySource:
+            auth = {'callback_url': 'https://example.com/callback'}
+
+            def serialize(self):
+                return {'provider': 'osf'}
+
+        source = DummySource()
+        request = {
+            'request': {
+                'method': 'GET',
+                'url': 'https://example.com/file',
+                'headers': {},
+            },
+            'referrer': {'url': None},
+            'tech': {'ua': 'test-agent', 'ip': '127.0.0.1'},
+        }
+
+        await remote_logging.log_to_callback(
+            'download_file',
+            source=source,
+            request=request,
+            completed=completed,
+        )
+
+        assert captured['payload']['action_meta']['completed'] is expected
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize('status_code', [200, 500, None])
+    async def test_download_action_forwards_status_code(self, monkeypatch, status_code):
+        """The status lets the OSF tell a real failure from a user cancelling mid-stream."""
+        captured = {}
+
+        async def fake_send_signed_request(method, url, payload):
+            captured['payload'] = payload
+            return 200, b'success'
+
+        monkeypatch.setattr(remote_logging.utils, 'send_signed_request', fake_send_signed_request)
+
+        class DummySource:
+            auth = {'callback_url': 'https://example.com/callback'}
+
+            def serialize(self):
+                return {'provider': 'osf'}
+
+        source = DummySource()
+        request = {
+            'request': {
+                'method': 'GET',
+                'url': 'https://example.com/file',
+                'headers': {},
+            },
+            'referrer': {'url': None},
+            'tech': {'ua': 'test-agent', 'ip': '127.0.0.1'},
+        }
+
+        await remote_logging.log_to_callback(
+            'download_zip',
+            source=source,
+            request=request,
+            completed=False,
+            status_code=status_code,
+        )
+
+        assert captured['payload']['action_meta']['status_code'] == status_code
+
+    @pytest.mark.asyncio
+    async def test_non_download_action_omits_completed_flag(self, monkeypatch):
+        captured = {}
+
+        async def fake_send_signed_request(method, url, payload):
+            captured['payload'] = payload
+            return 200, b'success'
+
+        monkeypatch.setattr(remote_logging.utils, 'send_signed_request', fake_send_signed_request)
+
+        class DummySource:
+            auth = {'callback_url': 'https://example.com/callback'}
+
+            def serialize(self):
+                return {'provider': 'osf'}
+
+        source = DummySource()
+        request = {
+            'request': {
+                'method': 'GET',
+                'url': 'https://example.com/file',
+                'headers': {},
+            },
+            'referrer': {'url': None},
+            'tech': {'ua': 'test-agent', 'ip': '127.0.0.1'},
+        }
+
+        await remote_logging.log_to_callback('create', source=source, request=request)
+
+        assert 'completed' not in captured['payload']['action_meta']
+
+    @pytest.mark.asyncio
+    async def test_download_action_forwards_link_tags(self, monkeypatch):
+        captured = {}
+
+        async def fake_send_signed_request(method, url, payload):
+            captured['payload'] = payload
+            return 200, b'success'
+
+        monkeypatch.setattr(remote_logging.utils, 'send_signed_request', fake_send_signed_request)
+
+        class DummySource:
+            auth = {'callback_url': 'https://example.com/callback'}
+
+            def serialize(self):
+                return {'provider': 'osf'}
+
+        source = DummySource()
+        request = {
+            'request': {
+                'method': 'GET',
+                'url': 'https://example.com/folder?zip=&source=files&tz=Europe%2FKyiv',
+                'headers': {},
+            },
+            'referrer': {'url': None},
+            'tech': {'ua': 'test-agent', 'ip': '127.0.0.1'},
+        }
+
+        await remote_logging.log_to_callback('download_zip', source=source, request=request)
+
+        assert captured['payload']['action_meta']['source'] == 'files'
+        assert captured['payload']['action_meta']['tz'] == 'Europe/Kyiv'
+
+    @pytest.mark.asyncio
+    async def test_download_action_omits_absent_link_tags(self, monkeypatch):
+        captured = {}
+
+        async def fake_send_signed_request(method, url, payload):
+            captured['payload'] = payload
+            return 200, b'success'
+
+        monkeypatch.setattr(remote_logging.utils, 'send_signed_request', fake_send_signed_request)
+
+        class DummySource:
+            auth = {'callback_url': 'https://example.com/callback'}
+
+            def serialize(self):
+                return {'provider': 'osf'}
+
+        source = DummySource()
+        request = {
+            'request': {
+                'method': 'GET',
+                'url': 'https://example.com/file',
+                'headers': {},
+            },
+            'referrer': {'url': None},
+            'tech': {'ua': 'test-agent', 'ip': '127.0.0.1'},
+        }
+
+        await remote_logging.log_to_callback('download_file', source=source, request=request)
+
+        assert 'source' not in captured['payload']['action_meta']
+        assert 'tz' not in captured['payload']['action_meta']
+
+    @pytest.mark.asyncio
+    async def test_download_link_tags_are_length_capped(self, monkeypatch):
+        captured = {}
+
+        async def fake_send_signed_request(method, url, payload):
+            captured['payload'] = payload
+            return 200, b'success'
+
+        monkeypatch.setattr(remote_logging.utils, 'send_signed_request', fake_send_signed_request)
+
+        class DummySource:
+            auth = {'callback_url': 'https://example.com/callback'}
+
+            def serialize(self):
+                return {'provider': 'osf'}
+
+        source = DummySource()
+        oversized = 'f' * (remote_logging.MAX_DOWNLOAD_TAG_LENGTH + 50)
+        request = {
+            'request': {
+                'method': 'GET',
+                'url': f'https://example.com/file?source={oversized}',
+                'headers': {},
+            },
+            'referrer': {'url': None},
+            'tech': {'ua': 'test-agent', 'ip': '127.0.0.1'},
+        }
+
+        await remote_logging.log_to_callback('download_file', source=source, request=request)
+
+        assert len(captured['payload']['action_meta']['source']) == \
+            remote_logging.MAX_DOWNLOAD_TAG_LENGTH
+
+
 class TestScrubPayloadForKeen:
 
     def test_flat_dict(self):

tests/server/api/v1/test_metadata_mixin.py

@@ -120,6 +120,20 @@ async def test_download_file_headers_no_stream_name(self, http_request, mock_str
 
         handler.write_stream.assert_awaited_once()
 
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize('write_stream_result, expected_completed', [(True, True), (False, False)])
+    async def test_download_file_records_stream_completion(self, http_request, mock_stream,
+                                                           write_stream_result, expected_completed):
+
+        handler = mock_handler(http_request)
+        handler.provider.download = MockCoroutine(return_value=mock_stream)
+        handler.path = WaterButlerPath('/test_file')
+        handler.write_stream = MockCoroutine(return_value=write_stream_result)
+
+        await handler.download_file()
+
+        assert handler._download_completed is expected_completed
+
     @pytest.mark.asyncio
     @pytest.mark.parametrize("given_arg,expected_name,filtered_name", [
         (['résumé.doc'], 'r%C3%A9sum%C3%A9.doc', 'resume.doc'),

tests/server/api/v1/test_provider.py

@@ -165,15 +165,18 @@ async def test_data_received_stream(self, http_request):
 class TestProviderHandlerFinish:
 
     @pytest.mark.asyncio
-    async def test_on_finish_download_file(self, http_request):
+    @pytest.mark.parametrize('download_completed, expected_completed', [(True, True), (False, False)])
+    async def test_on_finish_download_file(self, http_request, download_completed, expected_completed):
 
         handler = mock_handler(http_request)
         handler.request.method = 'GET'
         handler.path = WaterButlerPath('/file')
+        handler._download_completed = download_completed
         handler._send_hook = mock.Mock()
 
         assert handler.on_finish() is None
-        handler._send_hook.assert_called_once_with('download_file')
+        handler._send_hook.assert_called_once_with(
+            'download_file', completed=expected_completed, status_code=200)
 
     @pytest.mark.asyncio
     async def test_on_finish_download_zip(self, http_request):
@@ -185,7 +188,87 @@ async def test_on_finish_download_zip(self, http_request):
         handler._send_hook = mock.Mock()
 
         assert handler.on_finish() is None
-        handler._send_hook.assert_called_once_with('download_zip')
+        handler._send_hook.assert_called_once_with('download_zip', completed=True, status_code=200)
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize('status', [500, 502, 400, 404])
+    async def test_on_finish_failed_download_file(self, http_request, status):
+        """A file download that authorized and started but then errored is reported as
+        failed, so the OSF can count it -- the "attempted but failed" case."""
+        handler = mock_handler(http_request)
+        handler.request.method = 'GET'
+        handler.path = WaterButlerPath('/file')
+        handler._status_code = status
+        handler._send_hook = mock.Mock()
+
+        assert handler.on_finish() is None
+        handler._send_hook.assert_called_once_with(
+            'download_file', completed=False, status_code=status)
+
+    @pytest.mark.asyncio
+    async def test_on_finish_failed_download_zip(self, http_request):
+        handler = mock_handler(http_request)
+        handler.request.method = 'GET'
+        handler.request.query_arguments['zip'] = ''
+        handler.path = WaterButlerPath('/folder/')
+        handler._status_code = 500
+        handler._send_hook = mock.Mock()
+
+        assert handler.on_finish() is None
+        handler._send_hook.assert_called_once_with(
+            'download_zip', completed=False, status_code=500)
+
+    @pytest.mark.asyncio
+    async def test_failed_download_not_reported_without_a_provider(self, http_request):
+        """A request that failed during auth never got a provider, so there's no callback
+        url to report to -- it must stay silent, exactly as before."""
+        handler = mock_handler(http_request)
+        handler.request.method = 'GET'
+        handler.path = WaterButlerPath('/file')
+        handler.provider = None
+        handler._status_code = 500
+        handler._send_hook = mock.Mock()
+
+        assert handler.on_finish() is None
+        assert not handler._send_hook.called
+
+    @pytest.mark.asyncio
+    async def test_failed_download_not_reported_when_path_never_validated(self, http_request):
+        """self.path is still the raw string it starts as -- validation never finished, so
+        the download never really began."""
+        handler = mock_handler(http_request)
+        handler.request.method = 'GET'
+        handler.path = '/test_path'
+        handler._status_code = 500
+        handler._send_hook = mock.Mock()
+
+        assert handler.on_finish() is None
+        assert not handler._send_hook.called
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize('method', ['PUT', 'POST', 'DELETE'])
+    async def test_failed_non_download_is_not_reported(self, http_request, method):
+        """Only downloads are recorded on failure; a failed upload/move/delete stays silent."""
+        handler = mock_handler(http_request)
+        handler.request.method = method
+        handler.path = WaterButlerPath('/file')
+        handler._status_code = 500
+        handler._send_hook = mock.Mock()
+
+        assert handler.on_finish() is None
+        assert not handler._send_hook.called
+
+    @pytest.mark.asyncio
+    async def test_failed_folder_listing_is_not_reported(self, http_request):
+        """A folder GET without ?zip is a metadata listing, not a download."""
+        handler = mock_handler(http_request)
+        handler.request.method = 'GET'
+        handler.path = WaterButlerPath('/folder/')
+        handler._status_code = 500
+        handler._send_hook = mock.Mock()
+
+        assert handler.on_finish() is None
+        assert not handler._send_hook.called
 
     @pytest.mark.asyncio
     async def test_dont_send_hook_on_file_metadata(self, http_request):
@@ -336,4 +419,4 @@ async def test_logging_direct_partial_download_file(self, http_request):
         handler._send_hook = mock.Mock()
 
         assert handler.on_finish() is None
-        handler._send_hook.assert_called_once_with('download_file')
+        handler._send_hook.assert_called_once_with('download_file', completed=True, status_code=302)