From f08100355b61b12656ef3f0eac2bad308da57143 Mon Sep 17 00:00:00 2001
From: Zixian Cai <2891235+caizixian@users.noreply.github.com>
Date: Tue, 26 Aug 2025 11:28:12 +1000
Subject: [PATCH] Handle surrogate character error when hashing documents

---
 app/routes/document_routes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/routes/document_routes.py b/app/routes/document_routes.py
index 51218470..d17b5987 100644
--- a/app/routes/document_routes.py
+++ b/app/routes/document_routes.py
@@ -249,7 +249,7 @@ async def query_embeddings_by_file_id(
 
 
 def generate_digest(page_content: str):
-    hash_obj = hashlib.md5(page_content.encode())
+    hash_obj = hashlib.md5(page_content.encode('utf-8', errors="surrogateescape"))
     return hash_obj.hexdigest()