Invalidate CDN cache on plain document edits

config/settings/base.py

@@ -627,6 +627,8 @@
 CLOUDFLARE_API_KEY = env("CLOUDFLARE_API_KEY", default="")
 CLOUDFLARE_API_ZONE = env("CLOUDFLARE_API_ZONE", default="")
 CLOUDFLARE_HOSTS = env.list("CLOUDFLARE_HOSTS", default=[])
+# max operations per purge request (Business plan caps this at 100)
+CLOUDFLARE_PURGE_LIMIT = env.int("CLOUDFLARE_PURGE_LIMIT", default=100)
 
 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
 

documentcloud/documents/cache.py

@@ -0,0 +1,117 @@
+"""CDN cache invalidation for documents (CloudFront + Cloudflare)."""
+
+# Django
+from django.conf import settings
+
+# Standard Library
+import logging
+import uuid
+
+# Third Party
+import boto3
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class CloudflarePurgeError(requests.RequestException):
+    """A Cloudflare purge request was rejected.
+
+    Subclasses `RequestException` so the `invalidate_cache` task's
+    `autoretry_for=(RequestException,)` retries it, alongside the transport
+    errors `raise_for_status()` already raises.
+    """
+
+
+def _chunk(items, size):
+    """Yield successive `size`-length chunks of `items`."""
+    for i in range(0, len(items), size):
+        yield items[i : i + size]
+
+
+def _invalidate_cloudfront(paths):
+    """Invalidate the given paths from CloudFront in one batch."""
+    distribution_id = settings.CLOUDFRONT_DISTRIBUTION_ID
+    if not distribution_id or not paths:
+        return
+    cloudfront = boto3.client("cloudfront")
+    cloudfront.create_invalidation(
+        DistributionId=distribution_id,
+        InvalidationBatch={
+            "Paths": {"Quantity": len(paths), "Items": paths},
+            "CallerReference": str(uuid.uuid4()),
+        },
+    )
+
+
+def _invalidate_cloudflare(files=None, tags=None):
+    """Purge the given files and tags from Cloudflare.
+
+    `files` and `tags` cannot be combined in a single purge request (the zone
+    purge API is a `oneOf`), so they are sent as separate requests, each
+    chunked to the plan's per-request operation cap.
+    """
+    zone = settings.CLOUDFLARE_API_ZONE
+    if not zone:
+        return
+    url = f"https://api.cloudflare.com/client/v4/zones/{zone}/purge_cache"
+    headers = {
+        "X-Auth-Email": settings.CLOUDFLARE_API_EMAIL,
+        "X-Auth-Key": settings.CLOUDFLARE_API_KEY,
+    }
+    for key, values in (("files", files), ("tags", tags)):
+        for chunk in _chunk(values or [], settings.CLOUDFLARE_PURGE_LIMIT):
+            response = requests.post(
+                url, json={key: chunk}, headers=headers, timeout=10
+            )
+            # Cloudflare signals logical failures with `success: false` in a
+            # 200 body, so a clean status is not enough
+            if response.ok and response.json().get("success"):
+                continue
+            # a 429 is expected under burst load and handled by retry/backoff,
+            # so log it as a warning and keep real failures at error level
+            level = logging.WARNING if response.status_code == 429 else logging.ERROR
+            logger.log(
+                level,
+                "Cloudflare cache purge failed [%s]: status=%s body=%s",
+                key,
+                response.status_code,
+                response.text,
+            )
+            # raise so the Celery task retries (purging is idempotent):
+            # HTTPError for a bad status, CloudflarePurgeError for success=false
+            response.raise_for_status()
+            raise CloudflarePurgeError(response.text, response=response)
+
+
+def invalidate_cache_batch(documents):
+    """Invalidate the CloudFront and Cloudflare caches for many documents.
+
+    Cloudflare purges the API responses by Cache-Tag (`doc-{id}`) and the
+    frontend pages + public asset by URL; the two are mutually exclusive in a
+    single zone purge request, so they go in separate (chunked) requests.
+    CloudFront purges the underlying document file by path.
+    """
+    documents = list(documents)
+    if not documents:
+        return
+    logger.info("Invalidating cache for %s", [document.pk for document in documents])
+
+    cloudfront_paths = []
+    cloudflare_files = []
+    cloudflare_tags = []
+    for document in documents:
+        # the doc path without the s3 bucket name
+        doc_path = document.doc_path[document.doc_path.index("/") :]
+        cloudfront_paths.append(doc_path)
+        # always purge the frontend URLs: on a public -> private flip `access`
+        # is already private by now, but the public copy may still be cached at
+        # the edge - purging a URL that was never cached is harmless
+        cloudflare_files.extend(
+            host + document.get_absolute_url() for host in settings.CLOUDFLARE_HOSTS
+        )
+        cloudflare_files.append(settings.PUBLIC_ASSET_URL + doc_path[1:])
+        cloudflare_tags.append(document.cache_tag)
+
+    _invalidate_cloudfront(cloudfront_paths)
+    _invalidate_cloudflare(files=cloudflare_files, tags=cloudflare_tags)

documentcloud/documents/models/document.py

@@ -10,13 +10,10 @@
 import logging
 import sys
 import time
-import uuid
 from io import BytesIO
 
 # Third Party
-import boto3
 import pymupdf
-import requests
 from listcrunch import crunch, uncrunch
 from pikepdf import Page as PikePage, Pdf, Rectangle
 
@@ -283,13 +280,19 @@ def save(self, *args, **kwargs):
     @transaction.atomic
     def destroy(self):
         # DocumentCloud
-        from documentcloud.documents.tasks import delete_document_files, solr_delete
+        from documentcloud.documents.tasks import (
+            delete_document_files,
+            invalidate_cache,
+            solr_delete,
+        )
 
         self.status = Status.deleted
         self.save()
         DeletedDocument.objects.create(pk=self.pk)
         transaction.on_commit(lambda: delete_document_files.delay(self.path))
         transaction.on_commit(lambda: solr_delete.delay(self.pk))
+        # the CDN may still be serving the (now deleted) public copy
+        transaction.on_commit(lambda: invalidate_cache.delay(self.pk))
 
     @property
     def path(self):
@@ -701,51 +704,15 @@ def page_filter(text):
 
         return solr_document
 
-    def invalidate_cache(self):
-        """
-        Invalidate public CDN cache for this document's underlying file,
-        plus frontend URLs in Cloudflare
-        """
-        logger.info("Invalidating cache for %s", self.pk)
-        doc_path = self.doc_path[self.doc_path.index("/") :]
-
-        # cloudfront
-        distribution_id = settings.CLOUDFRONT_DISTRIBUTION_ID
-        if distribution_id:
-            # we want the doc path without the s3 bucket name
-            cloudfront = boto3.client("cloudfront")
-            cloudfront.create_invalidation(
-                DistributionId=distribution_id,
-                InvalidationBatch={
-                    "Paths": {"Quantity": 1, "Items": [doc_path]},
-                    "CallerReference": str(uuid.uuid4()),
-                },
-            )
-
-        # cloudflare
-        cloudflare_email = settings.CLOUDFLARE_API_EMAIL
-        cloudflare_key = settings.CLOUDFLARE_API_KEY
-        cloudflare_zone = settings.CLOUDFLARE_API_ZONE
-        asset_url = settings.PUBLIC_ASSET_URL + doc_path[1:]
+    @property
+    def cache_tag(self):
+        """The Cloudflare Cache-Tag marking this document's cached API responses.
 
-        if self.access == Access.public:
-            public_urls = [
-                host + self.get_absolute_url() for host in settings.CLOUDFLARE_HOSTS
-            ] + [asset_url]
-        else:
-            public_urls = [asset_url]
-
-        if cloudflare_zone:
-            requests.post(
-                "https://api.cloudflare.com/client/v4/zones/"
-                f"{cloudflare_zone}/purge_cache",
-                json={"files": public_urls},
-                headers={
-                    "X-Auth-Email": cloudflare_email,
-                    "X-Auth-Key": cloudflare_key,
-                },
-                timeout=10,
-            )
+        Purging this one tag clears the bare `/api/documents/{pk}/` URL and
+        every `?expand=…` / per-`Origin` variant at once - the key spaces a
+        URL purge can't enumerate.
+        """
+        return f"doc-{self.pk}"
 
     def index_on_commit(self, **kwargs):
         """Index the document in Solr on tranasction commit"""

documentcloud/documents/tasks.py

@@ -20,6 +20,7 @@
 from documentcloud.common.environment import httpsub, storage
 from documentcloud.core.choices import Language
 from documentcloud.documents import entity_extraction, modifications, solr
+from documentcloud.documents.cache import invalidate_cache_batch
 from documentcloud.documents.choices import Access, Status
 from documentcloud.documents.models import Document, DocumentError
 from documentcloud.documents.search import SOLR, SOLR_NOTES
@@ -411,13 +412,24 @@ def publish_scheduled_documents():
         document.index_on_commit(field_updates={"status": "set"})
 
 
-@shared_task
-def invalidate_cache(document_pk):
-    """Invalidate the CloudFront and CloudFlare caches"""
-    document = Document.objects.get(pk=document_pk)
-    document.invalidate_cache()
-    document.cache_dirty = False
-    document.save()
+@shared_task(autoretry_for=(RequestException,), retry_backoff=30)
+def invalidate_cache(*document_pks):
+    """Invalidate the CloudFront and CloudFlare caches for the given documents.
+
+    Variadic so the input is always iterable: `invalidate_cache.delay(pk)`
+    purges one document, `invalidate_cache.delay(*pks)` purges a batch in one
+    set of requests rather than one task per document.
+
+    Retries on Cloudflare request failures - purging is idempotent, and
+    `cache_dirty` is only cleared once the purge succeeds.
+    """
+    # only pk/slug are needed to build the purge URLs and tags - skip the
+    # heavy columns (page_spec, data, description, ...)
+    documents = list(Document.objects.filter(pk__in=document_pks).only("pk", "slug"))
+    invalidate_cache_batch(documents)
+    # clear the flag with a queryset update so we don't bump `updated_at` (an
+    # AutoLastModifiedField) - a cache purge is not a content change
+    Document.objects.filter(pk__in=document_pks).update(cache_dirty=False)
 
 
 # page modifications