Skip to content

Commit 57aa7fa

Browse files
committed
fix: set TCP_NODELAY on guardrails proxy to halve streaming overhead
1 parent f8d02fa commit 57aa7fa

1 file changed

Lines changed: 5 additions & 0 deletions

File tree

‎charts/pca-ai-serving/charts/pca-guardrails/files/guardrails_proxy.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
import json
2626
import os
27+
import socket
2728
import ssl
2829
import threading
2930
import time
@@ -324,6 +325,10 @@ def normalize_orchestrator_messages(body: dict) -> dict:
324325
class ProxyHandler(BaseHTTPRequestHandler):
325326
protocol_version = "HTTP/1.1"
326327

328+
def setup(self):
329+
super().setup()
330+
self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
331+
327332
def _write_payload(self, status: int, payload: bytes, content_type: str) -> None:
328333
self.send_response(status)
329334
self.send_header("Content-Type", content_type)

0 commit comments

Comments
 (0)