📒 fix: Ignore invalid utf-8 characters when handling PDFs

app/routes/document_routes.py

@@ -249,7 +249,12 @@ async def query_embeddings_by_file_id(
 
 
 def generate_digest(page_content: str):
-    hash_obj = hashlib.md5(page_content.encode('utf-8', errors="surrogateescape"))
+    try:
+        hash_obj = hashlib.md5(page_content.encode("utf-8"))
+    except UnicodeEncodeError:
+        hash_obj = hashlib.md5(
+            page_content.encode("utf-8", "ignore").decode("utf-8").encode("utf-8")
+        )
     return hash_obj.hexdigest()
 
 

app/utils/document_loader.py

@@ -147,6 +147,18 @@ def get_loader(filename: str, file_content_type: str, filepath: str):
 
 
 def clean_text(text: str) -> str:
+    """
+    Clean up text from PDF lopader
+
+    :param text: The original text
+    :return: Cleaned text
+    """
+    text = remove_null(text)
+    text = remove_non_utf8(text)
+    return text
+
+
+def remove_null(text: str) -> str:
     """
     Remove NUL (0x00) characters from a string.
 
@@ -156,6 +168,19 @@ def clean_text(text: str) -> str:
     return text.replace("\x00", "")
 
 
+def remove_non_utf8(text: str) -> str:
+    """
+    Remove invalid UTF-8 characters from a string, such as surrogate characters
+
+    :param text: The original text with potential invalid utf-8 characters
+    :return: Cleaned text without invalid utf-8 characters.
+    """
+    try:
+        return text.encode("utf-8", "ignore").decode("utf-8")
+    except UnicodeError:
+        return text
+
+
 def process_documents(documents: List[Document]) -> str:
     processed_text = ""
     last_page: Optional[int] = None