diff --git a/addons/base/views.py b/addons/base/views.py index be3342eb4d7..0f8312a5f69 100644 --- a/addons/base/views.py +++ b/addons/base/views.py @@ -56,7 +56,11 @@ DownloadEvent, ) from osf.utils import permissions -from osf.utils.download_telemetry import never_breaks_downloads, record_download +from osf.utils.download_telemetry import ( + classify_download_channel, + never_breaks_downloads, + record_download, +) from osf.external.gravy_valet import request_helpers from website.profile.utils import get_profile_image_url from website.project import decorators @@ -205,6 +209,11 @@ def _record_file_download(target, file_node, query_params, auth, version=None): if _download_request_is_from_mfr(query_params): return + source_area = query_params.get('source', '') + # a browser navigating a download link never sends an Authorization header; an + # API/OAuth client does, so a bearer token is a reliable "this is programmatic" signal + is_api_token = 'bearer' in request.headers.get('Authorization', '').lower() + record_download( download_type=DownloadEvent.FILE, resource_guid=getattr(target, '_id', '') or '', @@ -214,7 +223,8 @@ def _record_file_download(target, file_node, query_params, auth, version=None): user_guid=getattr(getattr(auth, 'user', None), '_id', None), ip=request.remote_addr, user_agent=request.headers.get('User-Agent', ''), - source_area=query_params.get('source', ''), + source_area=source_area, + download_channel=classify_download_channel(source_area, is_api_token=is_api_token), tz=query_params.get('tz', ''), ) @@ -237,6 +247,11 @@ def _record_zip_download(payload): # The provider root is the whole project; anything below it is one folder. is_whole_project = not materialized.strip('/') + source_area = action_meta.get('source', '') + # The WaterButler callback doesn't carry the original request's auth, so we can't tell + # an API zip from a crawler here — is_api_token stays False and those fold into OTHER. + # The source tag still tells frontend zips apart from the rest. + record_download( download_type=DownloadEvent.PROJECT if is_whole_project else DownloadEvent.FOLDER_ZIP, resource_guid=metadata.get('nid') or '', @@ -248,7 +263,8 @@ def _record_zip_download(payload): user_guid=(payload.get('auth') or {}).get('id'), ip=action_meta.get('ip'), user_agent=(payload.get('request_meta') or {}).get('user_agent', ''), - source_area=action_meta.get('source', ''), + source_area=source_area, + download_channel=classify_download_channel(source_area), tz=action_meta.get('tz', ''), ) diff --git a/admin/templates/admin/input_filter.html b/admin/templates/admin/input_filter.html new file mode 100644 index 00000000000..cf43ab2dab2 --- /dev/null +++ b/admin/templates/admin/input_filter.html @@ -0,0 +1,16 @@ +{% load i18n %} +

{% blocktranslate with filter_title=title %}By {{ filter_title }}{% endblocktranslate %}

+ diff --git a/admin/templates/download_events/download_events.html b/admin/templates/download_events/download_events.html index 10a5c46ad75..0a3e3edddce 100644 --- a/admin/templates/download_events/download_events.html +++ b/admin/templates/download_events/download_events.html @@ -67,11 +67,47 @@ th { font-size: 12px; color: #777; } td { padding: 8px 10px; border-bottom: 1px solid #505050; color: #aaa; } .empty-state { color: #777; font-size: 13px; } + .active-filters { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; + margin: 0 0 18px; + } + .active-filters-label { + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.04em; + color: #777; + } + .filter-chip { + font-size: 12px; + color: #cfcfcf; + background: #3a3a3a; + border: 1px solid #505050; + border-radius: 12px; + padding: 3px 10px; + } + .filter-chip .chip-label { color: #8f8f8f; } + .unknown-note { + font-size: 12px; + color: #9c9c9c; + margin: 10px 0 0; + } + .unknown-note strong { color: #cfcfcf; font-weight: 600; }

Download telemetry dashboard

Summaries and charts are scoped to the same filters and search terms as the table below.

+
+ Applied filters: + {% for applied in download_events_active_filters %} + {{ applied.label }}: {{ applied.value }} + {% empty %} + None — showing the full current range + {% endfor %} +
@@ -85,7 +121,7 @@

Download telemetry dashboard

Total GB
-
{{ download_events_dashboard.summary.total_gb|floatformat:2 }}
+
{{ download_events_dashboard.summary.total_gb|floatformat:2 }}{{ download_events_dashboard.summary.total_gb_tb_suffix }}
Unique users
@@ -93,7 +129,7 @@

Download telemetry dashboard

File vs zip (including whole-project zip) (GB)
-
{{ download_events_dashboard.split.file.gb|floatformat:2 }} / {{ download_events_dashboard.split.zip.gb|floatformat:2 }}
+
{{ download_events_dashboard.split.file.gb|floatformat:2 }}{{ download_events_dashboard.split.file.gb_tb_suffix }} / {{ download_events_dashboard.split.zip.gb|floatformat:2 }}{{ download_events_dashboard.split.zip.gb_tb_suffix }}
Failed zip downloads (server or provider error)
@@ -127,6 +163,15 @@

Download telemetry dashboard

+ {% with unknown=download_events_dashboard.storage_regions_unknown %} + {% if unknown %} +

+ Not shown (region unknown): {{ unknown.downloads }} downloads · + {{ unknown.gb|floatformat:2 }} GB + — Files: {{ unknown.file_count }} · Zips: {{ unknown.zip_count }} +

+ {% endif %} + {% endwith %}
@@ -140,6 +185,15 @@

Download telemetry dashboard

+ {% with unknown=download_events_dashboard.user_regions_unknown %} + {% if unknown %} +

+ Not shown (region unknown): {{ unknown.downloads }} downloads · + {{ unknown.gb|floatformat:2 }} GB + — Files: {{ unknown.file_count }} · Zips: {{ unknown.zip_count }} +

+ {% endif %} + {% endwith %}
@@ -224,6 +278,33 @@

+
+
By channel (frontend / API / other)
+ + + + + + + + + + {% for channel in download_events_dashboard.channels %} + + + + + + {% empty %} + + {% endfor %} + +
ChannelGBDownloads
{{ channel.name }}{{ channel.gb|floatformat:2 }}{{ channel.downloads }}
No activity in the current range.
+

+ Zips can only be split frontend vs. other — API zip downloads aren't + distinguishable at capture, so they fall under "Other / direct". +

+
By storage provider
@@ -246,6 +327,14 @@

{% endfor %}

+ {% with unknown=download_events_dashboard.storage_providers_unknown %} + {% if unknown %} +

+ No provider recorded: {{ unknown.downloads }} downloads · + {{ unknown.gb|floatformat:2 }} GB +

+ {% endif %} + {% endwith %}
@@ -561,6 +650,20 @@

} ); +// A zero slice ("File" or "Zip") has no arc, so don't pop a "File 0" / "Zip 0" +// tooltip for it — that reads as a bug when a filter zeroes one side out. +const doughnutOptions = { + ...commonOptions, + cutout: "60%", + plugins: { + ...commonOptions.plugins, + tooltip: { + ...commonOptions.plugins.tooltip, + filter: (item) => item.parsed !== 0 + } + } +}; + new Chart( document.getElementById("downloadsSplitChart"), { @@ -578,10 +681,7 @@

] }] }, - options: { - ...commonOptions, - cutout: "60%" - } + options: doughnutOptions } ); @@ -602,11 +702,7 @@

] }] }, - options: { - ...commonOptions, - cutout: "60%" - } - + options: doughnutOptions } ); diff --git a/osf/admin.py b/osf/admin.py index ebf9790300f..d2a38452a9a 100644 --- a/osf/admin.py +++ b/osf/admin.py @@ -445,6 +445,62 @@ def queryset(self, request, queryset): return queryset +class InputFilter(SimpleListFilter): + """A sidebar filter that takes a free-text value instead of a fixed list of choices. + + Django's list filters render a set of links; project and user are far too + high-cardinality for that. This renders a text box (see ``admin/input_filter.html``) + and hands the typed value to :meth:`queryset`. Subclasses set ``parameter_name`` / + ``title`` and implement ``queryset``. + """ + template = 'admin/input_filter.html' + + def lookups(self, request, model_admin): + # SimpleListFilter hides itself unless lookups() is non-empty; the value is + # unused because choices() is overridden below. + return ((),) + + def choices(self, changelist): + # keep every other active filter/search param when this box is submitted, so + # typing a project doesn't wipe the date range or an existing user filter + all_choice = next(super().choices(changelist)) + all_choice['query_parts'] = [ + (key, value) + for key, value in changelist.get_filters_params().items() + if key != self.parameter_name + ] + yield all_choice + + +class ProjectGuidFilter(InputFilter): + """Scope the dashboard to a single project/preprint by its guid.""" + + parameter_name = 'project_guid' + title = 'project guid' + + def queryset(self, request, queryset): + value = self.value() + if value: + return queryset.filter(resource_guid=value.strip()) + return queryset + + +class DownloadUserFilter(InputFilter): + """Scope the dashboard to a single user, matched by email (username) or user guid.""" + + parameter_name = 'download_user' + title = 'user (email or guid)' + + def queryset(self, request, queryset): + value = self.value() + if value: + term = value.strip() + return queryset.filter( + Q(user__username__iexact=term) | Q(user__guids___id=term) + ) + return queryset + + @admin.register(DownloadEvent) class DownloadEventsView(admin.ModelAdmin): change_list_template = 'download_events/download_events.html' @@ -462,6 +518,7 @@ class DownloadEventsView(admin.ModelAdmin): 'storage_region', 'ip', 'source_area', + 'channel_display', 'user_agent_display', 'created' ) @@ -472,7 +529,10 @@ class DownloadEventsView(admin.ModelAdmin): title='date and time (UTC)', ), ), + ProjectGuidFilter, + DownloadUserFilter, 'download_type', + 'download_channel', DownloadOutcomeFilter, 'zip_completed', 'storage_provider', @@ -519,6 +579,12 @@ def user_display(self, obj): """Sort the User column by the username (email) rather than the raw FK id.""" return obj.user or '—' + @admin.display(description='Channel', ordering='download_channel') + def channel_display(self, obj): + """Human-readable frontend/api/other. Blank ('—') for rows recorded before the + field existed and for zip downloads whose channel couldn't be resolved.""" + return obj.get_download_channel_display() if obj.download_channel else '—' + @admin.display(description='User agent', ordering='user_agent') def user_agent_display(self, obj): """Truncated in the table so it doesn't dominate the row; the full value is still @@ -566,8 +632,46 @@ def changelist_view(self, request, extra_context=None): extra_context = {} changelist = self.get_changelist_instance(request) extra_context['download_events_dashboard'] = self.get_dashboard_data(changelist.get_queryset(request)) + extra_context['download_events_active_filters'] = self._active_filters(request) return super().changelist_view(request, extra_context=extra_context) + # query-string param -> human label, for the "applied filters" banner above the charts + FILTER_LABELS = ( + ('q', 'Search'), + ('project_guid', 'Project'), + ('download_user', 'User'), + ('download_type', 'Download type'), + ('outcome', 'Outcome'), + ('zip_completed__exact', 'Zip completed'), + ('storage_provider', 'Storage provider'), + ) + + def _active_filters(self, request): + """The filters/search currently in effect, as ``[{'label', 'value'}]``, so the + dashboard can show at a glance what its numbers are scoped to. + + Purely presentational — it reads the same query string the changelist already + filtered on; it never changes what's queried. + """ + params = request.GET + active = [] + + # the date range arrives in date+time halves; recombine them into one readable line + date_from = ' '.join( + part for part in (params.get('created__range__gte_0'), params.get('created__range__gte_1')) if part + ) + date_to = ' '.join( + part for part in (params.get('created__range__lte_0'), params.get('created__range__lte_1')) if part + ) + if date_from or date_to: + active.append({'label': 'Date (UTC)', 'value': f"{date_from or '…'} → {date_to or 'now'}"}) + + for param, label in self.FILTER_LABELS: + value = params.get(param) + if value: + active.append({'label': label, 'value': value}) + return active + def _in_dashboard_group(self, request): """Membership in the allow-list group is the only key to this page. @@ -603,12 +707,28 @@ def has_change_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return False + # blank/null region, provider or user region all mean "we could not tell" + UNKNOWN_REGION_LABEL = 'Unknown' + # 1 TB = 1024 GB. GB stays the primary unit; TB is only shown once a figure is + # genuinely terabyte-scale, so small numbers aren't cluttered with "(0.0 TB)". + GB_PER_TB = 1024 + def _sum_bytes(self, queryset): return queryset.aggregate(total_bytes=Sum('size_bytes'))['total_bytes'] or 0 def _to_gb(self, total_bytes): return round((total_bytes or 0) / (1024**3), 2) + def _tb_suffix(self, gb): + """A parenthetical TB reading for terabyte-scale figures, e.g. ' (1.21 TB)'. + + Empty below 1 TB — GB is the primary unit and we don't want '(0.0 TB)' hanging + off every small number. + """ + if not gb or gb < self.GB_PER_TB: + return '' + return f' ({round(gb / self.GB_PER_TB, 2)} TB)' + def _percent(self, part, whole): """Empty ranges are normal — the default window is the last hour.""" if not whole: @@ -626,10 +746,12 @@ def get_dashboard_data(self, queryset): total_file_gb = self._to_gb(self._sum_bytes(file_queryset)) total_zip_gb = self._to_gb(self._sum_bytes(zip_queryset)) time_series = self._build_time_series(queryset) - storage_regions = self._build_region_breakdown(queryset, 'storage_region') - user_regions = self._build_region_breakdown(queryset, 'user_region') + # each breakdown returns the ranked *known* regions plus, separately, the + # "Unknown" bucket — so a large Unknown doesn't crowd real regions off the chart + storage_regions, storage_regions_unknown = self._build_region_breakdown(queryset, 'storage_region') + user_regions, user_regions_unknown = self._build_region_breakdown(queryset, 'user_region') # downloads and GB grouped by where the bytes came from (osfstorage vs addons) - storage_providers = self._build_region_breakdown(queryset, 'storage_provider') + storage_providers, storage_providers_unknown = self._build_region_breakdown(queryset, 'storage_provider') # Zip outcomes. Single files are recorded before any bytes move, so they have no # outcome and are left out of this breakdown entirely. @@ -649,12 +771,14 @@ def get_dashboard_data(self, queryset): 'file': { 'count': total_file_downloads, 'gb': total_file_gb, + 'gb_tb_suffix': self._tb_suffix(total_file_gb), 'count_percent': self._percent(total_file_downloads, total_downloads), 'gb_percent': self._percent(total_file_gb, total_gb), }, 'zip': { 'count': total_zip_downloads, 'gb': total_zip_gb, + 'gb_tb_suffix': self._tb_suffix(total_zip_gb), 'count_percent': self._percent(total_zip_downloads, total_downloads), 'gb_percent': self._percent(total_zip_gb, total_gb), }, @@ -664,6 +788,7 @@ def get_dashboard_data(self, queryset): 'summary': { 'total_downloads': total_downloads, 'total_gb': total_gb, + 'total_gb_tb_suffix': self._tb_suffix(total_gb), 'unique_users': queryset.exclude(user_id__isnull=True).values('user_id').distinct().count(), 'failed_zips': failed_zips, }, @@ -671,8 +796,12 @@ def get_dashboard_data(self, queryset): 'zip_outcomes': zip_outcomes, 'time_series': time_series, 'storage_regions': storage_regions, + 'storage_regions_unknown': storage_regions_unknown, 'storage_providers': storage_providers, + 'storage_providers_unknown': storage_providers_unknown, 'user_regions': user_regions, + 'user_regions_unknown': user_regions_unknown, + 'channels': self._build_channel_breakdown(queryset), 'top_projects': self._build_top_resource_breakdown(queryset), 'top_users': self._build_top_user_breakdown(queryset), } @@ -779,6 +908,12 @@ def _build_region_breakdown(self, queryset, field_name): `downloads` is the total request count; `file_count` and `zip_count` split it by request type (a zip is either a folder or a whole-project zip), so file + zip always equals the total. + + Returns ``(known_regions, unknown)``. "Unknown" (blank/null — we couldn't tell) is + pulled out of the ranked list and returned on its own, so a large Unknown bucket + doesn't crowd the real regions off the chart. The percentages scale to the largest + *known* region, so the country bars stay readable no matter how big Unknown is. + ``unknown`` is ``None`` when every row resolved to a real region. """ rows = queryset.values(field_name).annotate( downloads=Count('id'), @@ -790,18 +925,20 @@ def _build_region_breakdown(self, queryset, field_name): breakdown = defaultdict(lambda: {'downloads': 0, 'gb': 0.0, 'file_count': 0, 'zip_count': 0}) for row in rows: # blank and null both mean "we could not tell", so they fold together - region_name = (row[field_name] or 'Unknown').strip() or 'Unknown' + region_name = (row[field_name] or self.UNKNOWN_REGION_LABEL).strip() or self.UNKNOWN_REGION_LABEL breakdown[region_name]['downloads'] += row['downloads'] breakdown[region_name]['gb'] += (row['total_bytes'] or 0) / (1024**3) breakdown[region_name]['file_count'] += row['file_count'] breakdown[region_name]['zip_count'] += row['zip_count'] + unknown_data = breakdown.pop(self.UNKNOWN_REGION_LABEL, None) + # gb descending, then name ascending so the order is deterministic when GB ties # (and never depends on the incoming queryset's row order) ordered = sorted(breakdown.items(), key=lambda item: (-item[1]['gb'], item[0]))[:10] max_gb = max((data['gb'] for _, data in ordered), default=0) max_downloads = max((data['downloads'] for _, data in ordered), default=0) - return [ + known_regions = [ { 'name': name, 'downloads': data['downloads'], @@ -813,6 +950,48 @@ def _build_region_breakdown(self, queryset, field_name): } for name, data in ordered ] + unknown = None + if unknown_data: + unknown = { + 'name': self.UNKNOWN_REGION_LABEL, + 'downloads': unknown_data['downloads'], + 'file_count': unknown_data['file_count'], + 'zip_count': unknown_data['zip_count'], + 'gb': round(unknown_data['gb'], 2), + } + return known_regions, unknown + + def _build_channel_breakdown(self, queryset): + """Downloads + GB grouped by channel (frontend / api / other) — the explicit split + the User-Agent alone could never give us. Rows with no channel (recorded before the + field, or zips whose channel couldn't be resolved) are reported as 'Unknown'.""" + rows = queryset.values('download_channel').annotate( + downloads=Count('id'), + total_bytes=Sum('size_bytes'), + ) + labels = dict(DownloadEvent.DOWNLOAD_CHANNELS) + # Fold by channel in Python: a pre-existing annotation on the queryset (e.g. the + # sort's _outcome_rank) can leak into the GROUP BY and split a channel across rows, + # so sum them back together — same reason _build_region_breakdown folds. + folded = defaultdict(lambda: {'downloads': 0, 'bytes': 0}) + for row in rows: + channel = row['download_channel'] or '' + folded[channel]['downloads'] += row['downloads'] + folded[channel]['bytes'] += (row['total_bytes'] or 0) + + # frontend, api, other, then unknown last — a stable, meaningful order + rank = {DownloadEvent.FRONTEND: 0, DownloadEvent.API: 1, DownloadEvent.OTHER: 2, '': 3} + breakdown = [ + { + 'name': labels.get(channel, 'Unknown') if channel else 'Unknown', + 'channel': channel, + 'downloads': data['downloads'], + 'gb': self._to_gb(data['bytes']), + } + for channel, data in folded.items() + ] + breakdown.sort(key=lambda item: (rank.get(item['channel'], 4), item['name'])) + return breakdown def _build_top_resource_breakdown(self, queryset): rows = queryset.exclude(resource_guid='').values('resource_guid').annotate( diff --git a/osf/migrations/0052_downloadevent_download_channel.py b/osf/migrations/0052_downloadevent_download_channel.py new file mode 100644 index 00000000000..f7727a4df19 --- /dev/null +++ b/osf/migrations/0052_downloadevent_download_channel.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.26 on 2026-08-20 14:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('osf', '0051_alter_fileversion_options'), + ] + + operations = [ + migrations.AddField( + model_name='downloadevent', + name='download_channel', + field=models.CharField(blank=True, choices=[('frontend', 'Frontend (website)'), ('api', 'API client'), ('other', 'Other / direct')], default='', max_length=16), + ), + ] diff --git a/osf/models/download_event.py b/osf/models/download_event.py index f06a34c1b4c..06f4c8eac36 100644 --- a/osf/models/download_event.py +++ b/osf/models/download_event.py @@ -19,6 +19,20 @@ class DownloadEvent(models.Model): (PROJECT, 'Whole-project zip'), ) + # Where the download came from, decided server-side at capture (not from the + # User-Agent, which browsers and bots alike can set). FRONTEND = an OSF UI download + # link; API = an authenticated API/OAuth client; OTHER = a direct link or crawler. + # Blank for rows recorded before this field, and for zip downloads whose channel we + # can't fully resolve (see the capture code). + FRONTEND = 'frontend' + API = 'api' + OTHER = 'other' + DOWNLOAD_CHANNELS = ( + (FRONTEND, 'Frontend (website)'), + (API, 'API client'), + (OTHER, 'Other / direct'), + ) + created = models.DateTimeField(auto_now_add=True, db_index=True) # what was downloaded @@ -47,6 +61,11 @@ class DownloadEvent(models.Model): # the client's User-Agent, to tell frontend downloads apart from API clients and # crawlers. Blank when we couldn't read one (and for rows recorded before this field). user_agent = models.TextField(blank=True, default='') + # explicit frontend/api/other classification, set server-side at capture time. See the + # DOWNLOAD_CHANNELS constants above. + download_channel = models.CharField( + max_length=16, blank=True, default='', choices=DOWNLOAD_CHANNELS, + ) # nullable: anonymous downloads of public files user = models.ForeignKey( diff --git a/osf/utils/download_telemetry.py b/osf/utils/download_telemetry.py index f7fa0079070..e0a376d09ba 100644 --- a/osf/utils/download_telemetry.py +++ b/osf/utils/download_telemetry.py @@ -16,6 +16,34 @@ LOGGED_CONTEXT_KEYS = ('download_type', 'resource_guid', 'file_id', 'user_guid') +def classify_download_channel(source_area, is_api_token=False): + """Best server-side guess at where a download came from, for the dashboard's + frontend-vs-API split. + + Deliberately does NOT trust the User-Agent — browsers and bots alike set it (the QA + screenshots showed automated traffic sending ordinary Chrome UA strings). The signals + here are ones the client can't fake into looking like the OSF UI: + + - an API/OAuth bearer token means a programmatic client (a browser navigating a + download link never sends one) -> API + - otherwise a ``source`` tag means the request came from an OSF UI download link, + which only our own frontend adds -> FRONTEND + - everything else (direct links, crawlers) -> OTHER + + Only single-file downloads, captured at the redirect view, can see the token. Zip + downloads are captured from the WaterButler callback, which doesn't carry the original + request's auth, so their API traffic can't be told from OTHER — they still split + FRONTEND vs not by the source tag. ``is_api_token`` is always False for zips. + """ + from osf.models import DownloadEvent + + if is_api_token: + return DownloadEvent.API + if source_area: + return DownloadEvent.FRONTEND + return DownloadEvent.OTHER + + def never_breaks_downloads(fn): """Swallow and log anything this raises. @@ -65,6 +93,7 @@ def write_download_event( ip=None, user_agent='', source_area='', + download_channel='', tz='', ): """Resolve the expensive bits and write one row. @@ -103,6 +132,7 @@ def write_download_event( # capped: the User-Agent comes off the request, so it's client-controlled user_agent=_truncate(user_agent, 512), source_area=_truncate(source_area, 128), + download_channel=_truncate(download_channel, 16), user=user, ) diff --git a/tests/test_download_events_dashboard.py b/tests/test_download_events_dashboard.py index 94224a490d7..97cd8d307ba 100644 --- a/tests/test_download_events_dashboard.py +++ b/tests/test_download_events_dashboard.py @@ -6,7 +6,12 @@ from django.test import RequestFactory from django.utils import timezone -from osf.admin import DASHBOARD_GROUP_NAME, DownloadEventsView +from osf.admin import ( + DASHBOARD_GROUP_NAME, + DownloadEventsView, + ProjectGuidFilter, + DownloadUserFilter, +) from osf.models import DownloadEvent from osf_tests.factories import AuthUserFactory, ProjectFactory, PreprintFactory from tests.base import OsfTestCase @@ -201,9 +206,12 @@ def test_storage_provider_breakdown(self): def test_blank_storage_provider_folds_into_unknown(self): make_event(storage_provider='') - providers = self.admin.get_dashboard_data(DownloadEvent.objects.all())['storage_providers'] + data = self.admin.get_dashboard_data(DownloadEvent.objects.all()) - assert [row['name'] for row in providers] == ['Unknown'] + # Unknown is pulled out of the ranked list and reported on its own + assert data['storage_providers'] == [] + assert data['storage_providers_unknown']['name'] == 'Unknown' + assert data['storage_providers_unknown']['downloads'] == 1 def test_region_breakdown_splits_requests_by_type(self): # Germany: 2 files + 1 folder zip + 1 project zip = 4 total, 3 zips @@ -245,8 +253,82 @@ def test_blank_and_null_regions_fold_into_unknown(self): data = self.admin.get_dashboard_data(DownloadEvent.objects.all()) - assert [row['name'] for row in data['storage_regions']] == ['Unknown'] - assert data['storage_regions'][0]['downloads'] == 2 + # blank and whitespace-only both mean "we couldn't tell" and fold into the + # separate Unknown bucket, out of the ranked region list + assert data['storage_regions'] == [] + assert data['storage_regions_unknown']['name'] == 'Unknown' + assert data['storage_regions_unknown']['downloads'] == 2 + + def test_unknown_is_separated_from_known_regions(self): + """ENG: a large Unknown bucket must not crowd real regions off the chart. The + ranked list holds only known regions; Unknown is reported on its own and the + bars scale to the largest known region.""" + make_event(storage_region='Germany', size_bytes=5 * 1024 ** 3) + make_event(storage_region='', size_bytes=1024 ** 3) + make_event(storage_region='', size_bytes=1024 ** 3) + + data = self.admin.get_dashboard_data(DownloadEvent.objects.all()) + + assert [r['name'] for r in data['storage_regions']] == ['Germany'] + # scales to itself now that Unknown is out of the ranking + assert data['storage_regions'][0]['gb_percent'] == 100 + assert data['storage_regions_unknown']['downloads'] == 2 + assert data['storage_regions_unknown']['gb'] == 2 + + def test_no_unknown_bucket_when_every_region_resolves(self): + make_event(storage_region='Germany', size_bytes=1024 ** 3) + + data = self.admin.get_dashboard_data(DownloadEvent.objects.all()) + + assert data['storage_regions_unknown'] is None + + def test_tb_suffix_only_appears_at_terabyte_scale(self): + assert self.admin._tb_suffix(0) == '' + assert self.admin._tb_suffix(500) == '' + assert self.admin._tb_suffix(1023.9) == '' + assert self.admin._tb_suffix(1024) == ' (1.0 TB)' + assert self.admin._tb_suffix(2560) == ' (2.5 TB)' + + def test_total_gb_gets_a_tb_reading_when_terabyte_scale(self): + make_event(size_bytes=2 * 1024 ** 4) # 2 TB + + data = self.admin.get_dashboard_data(DownloadEvent.objects.all()) + + assert data['summary']['total_gb'] == 2048 + assert data['summary']['total_gb_tb_suffix'] == ' (2.0 TB)' + + def test_sub_terabyte_total_has_no_tb_reading(self): + make_event(size_bytes=3 * 1024 ** 3) # 3 GB + + data = self.admin.get_dashboard_data(DownloadEvent.objects.all()) + + assert data['summary']['total_gb_tb_suffix'] == '' + + def test_channel_breakdown_groups_by_channel(self): + make_event(download_channel=DownloadEvent.FRONTEND, size_bytes=2 * 1024 ** 3) + make_event(download_channel=DownloadEvent.FRONTEND, size_bytes=1024 ** 3) + make_event(download_channel=DownloadEvent.API, size_bytes=1024 ** 3) + make_event(download_channel='') # recorded before the field / unresolved + + channels = self.admin.get_dashboard_data(DownloadEvent.objects.all())['channels'] + by_name = {c['name']: c for c in channels} + + assert by_name['Frontend (website)']['downloads'] == 2 + assert by_name['Frontend (website)']['gb'] == 3 + assert by_name['API client']['downloads'] == 1 + assert by_name['Unknown']['downloads'] == 1 + + def test_channel_breakdown_orders_frontend_api_other_then_unknown(self): + make_event(download_channel=DownloadEvent.OTHER) + make_event(download_channel='') + make_event(download_channel=DownloadEvent.FRONTEND) + make_event(download_channel=DownloadEvent.API) + + channels = self.admin.get_dashboard_data(DownloadEvent.objects.all())['channels'] + + assert [c['name'] for c in channels] == [ + 'Frontend (website)', 'API client', 'Other / direct', 'Unknown', + ] def test_top_projects_shows_title_and_guid(self): user = AuthUserFactory() @@ -388,6 +470,16 @@ def test_user_agent_display_truncates_long_agents(self): def test_user_agent_display_falls_back_when_blank(self): assert self.admin.user_agent_display(make_event(user_agent='')) == '—' + def test_channel_column_declares_a_sort_field(self): + assert self.admin.channel_display.admin_order_field == 'download_channel' + + def test_channel_display_shows_the_human_label(self): + event = make_event(download_channel=DownloadEvent.API) + assert self.admin.channel_display(event) == 'API client' + + def test_channel_display_falls_back_when_blank(self): + assert self.admin.channel_display(make_event(download_channel='')) == '—' + def test_outcome_annotation_does_not_change_dashboard_numbers(self): """Production feeds get_queryset() (annotated with _outcome_rank for sorting) into get_dashboard_data. The annotation must not alter any aggregate — guards against a @@ -402,6 +494,108 @@ def test_outcome_annotation_does_not_change_dashboard_numbers(self): assert annotated == plain +class TestDashboardFilters(OsfTestCase): + """Project and user input filters (they also drive the charts, since the dashboard + reads the same filtered changelist queryset).""" + + def setUp(self): + super().setUp() + self.admin = DownloadEventsView(DownloadEvent, AdminSite()) + + def _make(self, filter_cls, params): + request = RequestFactory().get('/admin/osf/downloadevent/', params) + return filter_cls(request, dict(params), DownloadEvent, self.admin) + + def test_project_filter_scopes_to_one_guid(self): + node = ProjectFactory() + make_event(resource_guid=node._id) + make_event(resource_guid='someotherguid') + + f = self._make(ProjectGuidFilter, {'project_guid': node._id}) + result = f.queryset(None, DownloadEvent.objects.all()) + + assert list(result.values_list('resource_guid', flat=True)) == [node._id] + + def test_project_filter_ignores_surrounding_whitespace(self): + node = ProjectFactory() + make_event(resource_guid=node._id) + + f = self._make(ProjectGuidFilter, {'project_guid': f' {node._id} '}) + result = f.queryset(None, DownloadEvent.objects.all()) + + assert result.count() == 1 + + def test_user_filter_matches_by_email(self): + user = AuthUserFactory() + make_event(user=user) + make_event(user=AuthUserFactory()) + + f = self._make(DownloadUserFilter, {'download_user': user.username}) + result = f.queryset(None, DownloadEvent.objects.all()) + + assert list(result.values_list('user_id', flat=True)) == [user.id] + + def test_user_filter_matches_by_guid(self): + user = AuthUserFactory() + make_event(user=user) + make_event(user=AuthUserFactory()) + + f = self._make(DownloadUserFilter, {'download_user': user._id}) + result = f.queryset(None, DownloadEvent.objects.all()) + + assert list(result.values_list('user_id', flat=True)) == [user.id] + + def test_blank_filter_value_is_a_noop(self): + make_event(resource_guid='a') + make_event(resource_guid='b') + + f = self._make(ProjectGuidFilter, {}) + result = f.queryset(None, DownloadEvent.objects.all()) + + assert result.count() == 2 + + +class TestActiveFiltersBanner(OsfTestCase): + """The read-only summary of what the dashboard is currently scoped to.""" + + def setUp(self): + super().setUp() + self.admin = DownloadEventsView(DownloadEvent, AdminSite()) + + def test_lists_applied_filters_with_labels(self): + request = RequestFactory().get('/', { + 'project_guid': 'abcde', + 'download_user': 'a@b.com', + 'download_type': 'file', + 'q': 'chrome', + }) + + active = {f['label']: f['value'] for f in self.admin._active_filters(request)} + + assert active['Project'] == 'abcde' + assert active['User'] == 'a@b.com' + assert active['Download type'] == 'file' + assert active['Search'] == 'chrome' + + def test_combines_the_date_range_halves(self): + request = RequestFactory().get('/', { + 'created__range__gte_0': '2026-01-01', + 'created__range__gte_1': '00:00:00', + 'created__range__lte_0': '2026-01-02', + 'created__range__lte_1': '12:00:00', + }) + + active = self.admin._active_filters(request) + date = next(f for f in active if f['label'] == 'Date (UTC)') + + assert date['value'] == '2026-01-01 00:00:00 → 2026-01-02 12:00:00' + + def test_empty_when_no_filters_applied(self): + request = RequestFactory().get('/') + + assert self.admin._active_filters(request) == [] + + class TestStaffAccessMigration(OsfTestCase): """Django's admin rejects anyone without `is_staff` before our gate runs, so the allow-listed users need it to reach the page at all.""" diff --git a/tests/test_download_telemetry.py b/tests/test_download_telemetry.py index 74a4e2b1719..652c7561db0 100644 --- a/tests/test_download_telemetry.py +++ b/tests/test_download_telemetry.py @@ -10,7 +10,11 @@ from api_tests.utils import create_test_file from framework.auth import signing from osf.models import DownloadEvent, OSFUser -from osf.utils.download_telemetry import derive_user_region, record_download +from osf.utils.download_telemetry import ( + classify_download_channel, + derive_user_region, + record_download, +) from osf_tests.factories import AuthUserFactory, ProjectFactory from tests.base import OsfTestCase @@ -239,6 +243,19 @@ def test_missing_request_meta_leaves_user_agent_blank(self): assert res.status_code == 200 assert DownloadEvent.objects.get().user_agent == '' + def test_zip_with_a_source_tag_is_a_frontend_download(self): + # build_payload sends source='files' by default, i.e. from a UI link + self.app.put(self.url, json=self.build_payload()) + + assert DownloadEvent.objects.get().download_channel == DownloadEvent.FRONTEND + + def test_zip_without_a_source_tag_is_other(self): + """No source tag on a zip means it didn't come from a UI link. The callback can't + see the auth, so API vs crawler folds into 'other'.""" + self.app.put(self.url, json=self.build_payload(source='')) + + assert DownloadEvent.objects.get().download_channel == DownloadEvent.OTHER + def test_callback_still_succeeds_when_recording_fails(self, ): with pytest.MonkeyPatch.context() as patch: patch.setattr( @@ -309,6 +326,42 @@ def test_zip_completed_is_unset_for_single_files(self): assert DownloadEvent.objects.get().zip_completed is None + def test_source_tag_marks_a_frontend_download(self): + self.app.get(f'/download/{self.guid}/?source=file-detail', auth=self.user.auth) + + assert DownloadEvent.objects.get().download_channel == DownloadEvent.FRONTEND + + def test_no_source_or_token_is_other(self): + """A raw download URL with no UI source tag and no API token — a direct link.""" + self.app.get(f'/download/{self.guid}/', auth=self.user.auth) + + assert DownloadEvent.objects.get().download_channel == DownloadEvent.OTHER + + def test_bearer_token_marks_an_api_download(self): + """An API/OAuth client sends an Authorization: Bearer header; a browser navigating + a download link never does, so it's a reliable 'programmatic' signal.""" + self.node.is_public = True + self.node.save() + + class _UnauthenticatedCasResponse: + authenticated = False + user = None + + class _FakeCasClient: + def profile(self, token): + return _UnauthenticatedCasResponse() + + with pytest.MonkeyPatch.context() as patch: + # a bearer header makes OSF validate the token against CAS; stub it so the test + # doesn't reach the network (the token is fake, the download is public anyway) + patch.setattr('framework.auth.cas.get_client', lambda: _FakeCasClient()) + self.app.get( + f'/download/{self.guid}/', + headers={'Authorization': 'Bearer notarealtoken'}, + ) + + assert DownloadEvent.objects.get().download_channel == DownloadEvent.API + def test_anonymous_download_is_recorded_without_a_user(self): self.node.is_public = True self.node.save() @@ -364,6 +417,22 @@ def test_unknown_is_empty(self): assert derive_user_region('', None, '') == '' +class TestChannelClassification: + """The server-side frontend/api/other decision — deliberately not UA-based.""" + + def test_a_bearer_token_is_an_api_client(self): + assert classify_download_channel('files', is_api_token=True) == DownloadEvent.API + # a token wins even if a source tag is also present + assert classify_download_channel('', is_api_token=True) == DownloadEvent.API + + def test_a_source_tag_without_a_token_is_frontend(self): + assert classify_download_channel('file-detail') == DownloadEvent.FRONTEND + + def test_no_token_and_no_source_is_other(self): + assert classify_download_channel('') == DownloadEvent.OTHER + assert classify_download_channel('', is_api_token=False) == DownloadEvent.OTHER + + @pytest.mark.django_db class TestRecordDownloadNeverRaises: