Switch manual GitHub API calls to PyGithub

package.json

@@ -48,7 +48,7 @@
     "react-router": "7.13.2",
     "react-table-6": "6.11.0",
     "react-tabs": "6.1.1",
-    "react-virtuoso": "4.18.9",
+    "react-virtuoso": "4.18.10",
     "redoc": "2.4.0",
     "stream-browserify": "3.0.0",
     "styled-components": "6.1.19",

requirements/common.in

@@ -4,15 +4,15 @@ whitenoise[brotli]==6.12.0  # Used by Whitenoise to provide Brotli-compressed ve
 Django==6.0.3
 celery==5.6.3  # celery needed for data ingestion
 simplejson==3.20.2  # import simplejson
-newrelic==11.2.0
+newrelic==13.2.0
 certifi==2026.1.4
 
 psycopg[binary]==3.3.3
 
 jsonschema==4.26.0  # import jsonschema
 djangorestframework==3.17.0  # Imported as rest_framework
 django-cors-headers==4.9.0  # Listed as 3rd party app on settings.py
-drf-spectacular==0.29.0  # Used for REST API Schema Generation
+drf-spectacular==0.30.0  # Used for REST API Schema Generation
 mozlog==8.0.0
 
 # Used directly and also by Django's YAML serializer.
@@ -40,6 +40,9 @@ mozci[cache]==2.4.3
 # Dockerflow/CloudOps APIs
 dockerflow==2026.3.4
 
+# Structured logging to GCP Cloud Logging (task_id/run_id labels)
+google-cloud-logging==3.16.0
+
 # Measuring noise of perf data
 moz-measure-noise==2.70.0
 

tests/changelog/test_collector.py

@@ -3,75 +3,65 @@
 import os
 import re
 from datetime import datetime, timedelta
+from unittest.mock import MagicMock
 
 import responses
 
 from treeherder.changelog.collector import collect
+from treeherder.utils import github
 
 
 def random_id():
     return binascii.hexlify(os.urandom(16)).decode("utf8")
 
 
-RELEASES = re.compile(r"https://api.github.com/repos/.*/.*/releases.*")
-COMMITS = re.compile(r"https://api.github.com/repos/.*/.*/commits\?.*")
-COMMIT_INFO = re.compile(r"https://api.github.com/repos/.*/.*/commits/.*")
+def prepare_responses():
+    # Placeholder for backward compatibility with tests/changelog/test_tasks.py
+    pass
 
 
-def prepare_responses():
-    now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
-
-    def releases(request):
-        data = [
-            {
-                "name": "ok",
-                "published_at": now,
-                "id": random_id(),
-                "html_url": "url",
-                "tag_name": "some tag",
-                "author": {"login": "tarek"},
-            }
-        ]
-        return 200, {}, json.dumps(data)
-
-    responses.add_callback(
-        responses.GET, RELEASES, callback=releases, content_type="application/json"
-    )
-
-    def _commit():
-        files = [{"filename": "file1"}, {"filename": "file2"}]
-        return {
-            "files": files,
-            "name": "ok",
-            "sha": random_id(),
-            "html_url": "url",
-            "tag_name": "some tag",
-            "commit": {
-                "message": "yeah",
-                "author": {"name": "tarek", "date": now},
-                "files": files,
-            },
-        }
-
-    def commit(request):
-        return 200, {}, json.dumps(_commit())
-
-    def commits(request):
-        return 200, {}, json.dumps([_commit()])
-
-    responses.add_callback(
-        responses.GET, COMMITS, callback=commits, content_type="application/json"
-    )
-    responses.add_callback(
-        responses.GET, COMMIT_INFO, callback=commit, content_type="application/json"
-    )
-
-
-@responses.activate
-def test_collect():
+def mock_github(monkeypatch):
+    now = datetime.now()
+
+    def mock_get_repo(owner, repo_name):
+        mock_repo = MagicMock()
+        mock_repo.full_name = f"{owner}/{repo_name}"
+        mock_repo.name = repo_name
+
+        # Mock releases
+        mock_release = MagicMock()
+        mock_release.name = "ok"
+        mock_release.tag_name = "some tag"
+        mock_release.published_at = now
+        mock_release.id = 12345
+        mock_release.html_url = "url"
+        mock_release.author.login = "tarek"
+        mock_repo.get_releases.return_value = [mock_release]
+
+        # Mock commits
+        mock_commit_obj = MagicMock()
+        mock_commit_obj.sha = random_id()
+        mock_commit_obj.html_url = "url"
+        mock_commit_obj.commit.message = "yeah"
+        mock_commit_obj.commit.author.name = "tarek"
+        mock_commit_obj.commit.author.date = now
+
+        mock_file = MagicMock()
+        mock_file.filename = "config/config.yml"
+        mock_commit_obj.files = [mock_file]
+
+        mock_repo.get_commits.return_value = [mock_commit_obj]
+        mock_repo.get_commit.return_value = mock_commit_obj
+
+        return mock_repo
+
+    monkeypatch.setattr(github, "get_repo", mock_get_repo)
+
+
+def test_collect(monkeypatch):
     yesterday = datetime.now() - timedelta(days=1)
     yesterday = yesterday.strftime("%Y-%m-%dT%H:%M:%S")
-    prepare_responses()
+    mock_github(monkeypatch)
     res = list(collect(yesterday))
 
     # we're not looking into much details here, we can do this

tests/conftest.py

@@ -6,6 +6,7 @@
 import time
 from os.path import dirname, join
 from unittest.mock import MagicMock
+from urllib.parse import urlparse, urlunparse
 
 import kombu
 import moz_measure_noise
@@ -15,6 +16,10 @@
 from _pytest.monkeypatch import MonkeyPatch
 from django.conf import settings
 from django.core.management import call_command
+
+# Importing this module registers the ``setting_changed`` receiver that resets
+# Django's cache handler when ``settings.CACHES`` changes (see pytest_configure).
+from django.test.signals import setting_changed
 from rest_framework.test import APIClient
 
 import treeherder.etl.bugzilla
@@ -43,6 +48,53 @@ def pytest_addoption(parser):
     )
 
 
+# Redis exposes 16 logical databases (0-15) by default.
+REDIS_DB_COUNT = 16
+
+
+def redis_url_for_worker(base_url, worker_id):
+    """
+    Return ``base_url`` pointing at a per-xdist-worker Redis logical database.
+
+    Sessions and other cached data live in a single shared Redis instance, and
+    ``pytest_runtest_setup`` clears the cache (a Redis ``FLUSHDB``) before every
+    test. Under ``-n auto`` that means one worker's flush wipes another worker's
+    in-flight session, which intermittently breaks the auth session tests (see
+    Bug 2051952). Giving each worker its own database keeps a ``FLUSHDB`` local
+    to the worker that issued it.
+
+    Non-Redis locations (and an empty ``worker_id``) are returned unchanged.
+    """
+    if not worker_id or not isinstance(base_url, str):
+        return base_url
+    if not base_url.startswith(("redis://", "rediss://")):
+        return base_url
+
+    # xdist worker ids look like "gw0", "gw1", ...
+    db_index = int(worker_id.removeprefix("gw")) % REDIS_DB_COUNT
+    return urlunparse(urlparse(base_url)._replace(path=f"/{db_index}"))
+
+
+def pytest_configure(config):
+    """
+    Isolate each xdist worker's Redis cache into its own logical database so the
+    per-test ``cache.clear()`` cannot wipe another worker's session mid-request.
+    """
+    worker_id = os.environ.get("PYTEST_XDIST_WORKER")
+    if not worker_id:
+        return
+
+    caches = copy.deepcopy(settings.CACHES)
+    for cache_config in caches.values():
+        cache_config["LOCATION"] = redis_url_for_worker(cache_config.get("LOCATION"), worker_id)
+
+    settings.CACHES = caches
+    # Force Django's cache handler to drop its cached settings/connections so the
+    # rewritten LOCATION takes effect. This is the same reset Django performs for
+    # ``override_settings(CACHES=...)``.
+    setting_changed.send(sender=None, setting="CACHES", value=caches, enter=True)
+
+
 def pytest_runtest_setup(item):
     """
     Per-test setup.

tests/etl/taskcluster_pulse/test_handler.py

@@ -1,6 +1,8 @@
 import pytest
 
+from treeherder.etl.taskcluster_pulse import handler as tc_handler
 from treeherder.etl.taskcluster_pulse.handler import handle_message, handle_task_defined
+from treeherder.utils.logging_context import get_log_labels
 
 
 @pytest.mark.asyncio
@@ -43,6 +45,45 @@ async def test_handle_message_routes_task_defined():
     assert result[0]["result"] == "unknown"
 
 
+@pytest.mark.asyncio
+async def test_handle_message_sets_log_context(monkeypatch):
+    """handle_message wraps processing in a log_context carrying task_id/run_id."""
+    captured = {}
+
+    def capture_labels(*args, **kwargs):
+        captured.update(get_log_labels())
+        return {"state": "unscheduled", "result": "unknown"}
+
+    monkeypatch.setattr(tc_handler, "handle_task_defined", capture_labels)
+
+    task = {
+        "metadata": {"name": "t", "description": "d", "owner": "o@example.com"},
+        "created": "2025-01-01T00:00:00.000Z",
+        "workerType": "test-worker",
+        "tags": {},
+        "routes": ["tc-treeherder.v2.autoland.abc123"],
+        "extra": {"treeherder": {"symbol": "T", "tier": 1}},
+    }
+    message = {
+        "exchange": "exchange/taskcluster-queue/v1/task-defined",
+        "root_url": "https://firefox-ci-tc.services.mozilla.com",
+        "payload": {
+            "runId": 0,
+            "status": {"taskId": "AJBb7wqZT6K9kz4niYAatg", "state": "unscheduled", "runs": []},
+        },
+    }
+
+    await handle_message(message, task)
+
+    assert captured == {
+        "task_id": "AJBb7wqZT6K9kz4niYAatg",
+        "run_id": "0",
+        "component": "ingestion",
+    }
+    # context is cleaned up after the handler returns
+    assert get_log_labels() == {}
+
+
 def test_handle_task_defined():
     push_info = {
         "project": "autoland",