Switch manual GitHub API calls to PyGithub

tests/changelog/test_collector.py

@@ -3,75 +3,68 @@
 import os
 import re
 from datetime import datetime, timedelta
+from unittest.mock import MagicMock
 
 import responses
 
 from treeherder.changelog.collector import collect
+from treeherder.utils import github
 
 
 def random_id():
     return binascii.hexlify(os.urandom(16)).decode("utf8")
 
 
-RELEASES = re.compile(r"https://api.github.com/repos/.*/.*/releases.*")
-COMMITS = re.compile(r"https://api.github.com/repos/.*/.*/commits\?.*")
-COMMIT_INFO = re.compile(r"https://api.github.com/repos/.*/.*/commits/.*")
+def mock_github(monkeypatch):
+    now = datetime.now()
+    now_str = now.strftime("%Y-%m-%dT%H:%M:%SZ")
 
+    def mock_get_repo(owner_repo):
+        owner, repo_name = owner_repo.split("/")
+        mock_repo = MagicMock()
+        mock_repo.full_name = owner_repo
+        mock_repo.name = repo_name
 
-def prepare_responses():
-    now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
-
-    def releases(request):
-        data = [
-            {
-                "name": "ok",
-                "published_at": now,
-                "id": random_id(),
-                "html_url": "url",
-                "tag_name": "some tag",
-                "author": {"login": "tarek"},
-            }
-        ]
-        return 200, {}, json.dumps(data)
-
-    responses.add_callback(
-        responses.GET, RELEASES, callback=releases, content_type="application/json"
-    )
-
-    def _commit():
-        files = [{"filename": "file1"}, {"filename": "file2"}]
-        return {
-            "files": files,
+        # Mock releases
+        mock_release = MagicMock()
+        mock_release.raw_data = {
             "name": "ok",
-            "sha": random_id(),
+            "published_at": now_str,
+            "id": random_id(),
             "html_url": "url",
             "tag_name": "some tag",
+            "author": {"login": "tarek"},
+        }
+        mock_release.published_at = now
+        mock_repo.get_releases.return_value = [mock_release]
+
+        # Mock commits
+        mock_commit_obj = MagicMock()
+        mock_commit_obj.sha = random_id()
+        mock_commit_obj.html_url = "url"
+        mock_commit_obj.commit.message = "yeah"
+        mock_commit_obj.commit.author.raw_data = {"name": "tarek", "date": now_str}
+        mock_commit_obj.raw_data = {
+            "sha": mock_commit_obj.sha,
+            "html_url": mock_commit_obj.html_url,
             "commit": {
                 "message": "yeah",
-                "author": {"name": "tarek", "date": now},
-                "files": files,
+                "author": {"name": "tarek", "date": now_str},
             },
+            "files": [{"filename": "config/config.yml"}],
         }
+        mock_repo.get_commits.return_value = [mock_commit_obj]
+        mock_repo.get_commit.return_value = mock_commit_obj
 
-    def commit(request):
-        return 200, {}, json.dumps(_commit())
-
-    def commits(request):
-        return 200, {}, json.dumps([_commit()])
+        return mock_repo
 
-    responses.add_callback(
-        responses.GET, COMMITS, callback=commits, content_type="application/json"
-    )
-    responses.add_callback(
-        responses.GET, COMMIT_INFO, callback=commit, content_type="application/json"
-    )
+    monkeypatch.setattr(github.github, "get_repo", mock_get_repo)
 
 
-@responses.activate
-def test_collect():
+def test_collect(monkeypatch):
     yesterday = datetime.now() - timedelta(days=1)
     yesterday = yesterday.strftime("%Y-%m-%dT%H:%M:%S")
-    prepare_responses()
+    mock_github(monkeypatch)
     res = list(collect(yesterday))
 
     # we're not looking into much details here, we can do this

tests/etl/test_ingest_command.py

@@ -1,3 +1,4 @@
+from unittest.mock import MagicMock
 from treeherder.etl.management.commands import ingest
 
 REPO_META = {
@@ -9,56 +10,58 @@
 }
 
 
-def test_query_data_consumes_compare_dict(monkeypatch):
-    """query_data must read the GitHub compare REST response as a dict.
+def test_query_data_pygithub(monkeypatch):
+    """query_data must use PyGithub objects.
 
-    Regression guard for Bug 2009865, which switched ``compare_shas`` to return
-    a list of PyGithub commit objects (for the Pulse push loader) but left
-    query_data doing dict access (``.get("merge_base_commit")`` / ``["commits"]``),
-    breaking the ``ingest push`` command for GitHub repos.
+    This replaces test_query_data_consumes_compare_dict which was a regression guard
+    for Bug 2009865. Since we now use PyGithub objects throughout query_data,
+    we mock the repository and its methods.
     """
-    compare_by_range = {
-        # base branch vs head: the head isn't on the base branch, so the API
-        # reports a merge base whose parent is the real fork point.
-        "main...HEAD": {
-            "merge_base_commit": {
-                "sha": "BASE",
-                "commit": {"committer": {"date": "2026-01-01T00:00:00Z"}},
-                "parents": [
-                    {
-                        "sha": "PARENT",
-                        "url": "https://api.github.com/repos/o/r/commits/PARENT",
-                    }
-                ],
-            },
-            "commits": [],
-        },
-        # re-compare with the corrected base yields the push's commits
-        "PARENT...HEAD": {
-            "merge_base_commit": {"sha": "PARENT", "parents": []},
-            "commits": [
-                {
-                    "sha": "C1",
-                    "commit": {
-                        "message": "Fix the thing",
-                        "author": {"name": "Dev", "email": "dev@example.com"},
-                        "committer": {"date": "2026-02-02T00:00:00Z"},
-                    },
-                }
-            ],
-        },
-    }
+    mock_repo = MagicMock()
 
-    def fake_fetch_api(path, params=None):
-        return compare_by_range[path.split("/compare/")[1]]
+    # Define mock commits
+    c1 = MagicMock()
+    c1.sha = "C1"
+    c1.commit.message = "Fix the thing"
+    c1.commit.author.raw_data = {"name": "Dev", "email": "dev@example.com"}
+    c1.commit.committer.raw_data = {"date": "2026-02-02T00:00:00Z"}
 
-    def fake_fetch_api_full_url(url, params=None):
-        # The merge-base parent, with a committer date different from the merge
-        # base so query_data takes the simple (non-recursive) branch.
-        return {"sha": "PARENT", "commit": {"committer": {"date": "2026-02-02T00:00:00Z"}}}
+    # Define comparison results
+    comp1 = MagicMock()
+    # merge_base_commit
+    mb1 = MagicMock()
+    mb1.sha = "BASE"
+    mb1.commit.committer.raw_data = {"date": "2026-01-01T00:00:00Z"}
+    p1 = MagicMock()
+    p1.sha = "PARENT"
+    mb1.parents = [p1]
+    comp1.merge_base_commit = mb1
+    comp1.commits = []
 
-    monkeypatch.setattr(ingest, "fetch_api", fake_fetch_api)
-    monkeypatch.setattr(ingest, "fetch_api_full_url", fake_fetch_api_full_url)
+    comp2 = MagicMock()
+    mb2 = MagicMock()
+    mb2.sha = "PARENT"
+    mb2.parents = []
+    comp2.merge_base_commit = mb2
+    comp2.commits = [c1]
+
+    # Mock get_commit for the parent
+    parent_commit = MagicMock()
+    parent_commit.sha = "PARENT"
+    parent_commit.commit.committer.raw_data = {"date": "2026-02-02T00:00:00Z"}
+
+    def fake_compare(base, head):
+        if base == "main" and head == "HEAD":
+            return comp1
+        if base == "PARENT" and head == "HEAD":
+            return comp2
+        raise ValueError(f"Unexpected compare call: {base}...{head}")
+
+    mock_repo.compare.side_effect = fake_compare
+    mock_repo.get_commit.return_value = parent_commit
+
+    mock_get_repo = MagicMock(return_value=mock_repo)
+    monkeypatch.setattr(ingest.github, "get_repo", mock_get_repo)
 
     event_base_sha, commits = ingest.query_data(REPO_META, "HEAD")
 

treeherder/changelog/collector.py

@@ -23,11 +23,11 @@ def get_changes(self, **kw):
         filters = kw.get("filters")
         gh_options = {"number": kw.get("number", MAX_ITEMS)}
 
+        if "since" in kw:
+            gh_options["since"] = kw["since"]
+
         for release in github.get_releases(owner, repository, params=gh_options):
             release["files"] = []
-            # no "since" option for releases() we filter manually here
-            if "since" in kw and release["published_at"] <= kw["since"]:
-                continue
             name = release["name"] or release["tag_name"]
             yield {
                 "date": release["published_at"],