Add base64 chunk encoding (BTC_TX64, protocol v1.1)

btcmesh_cli.py

@@ -1,8 +1,15 @@
 #!/usr/bin/env python3
+import base64
 import uuid
 import os
 from core.config_loader import get_meshtastic_serial_port
 from core.logger_setup import setup_logger
+from core.reassembler import (
+    CHUNK_PREFIX,
+    CHUNK_PREFIX_B64,
+    ENCODING_B64,
+    ENCODING_HEX,
+)
 import time
 from pubsub import pub
 import queue
@@ -17,14 +24,30 @@ def is_valid_hex(s):
         return False
 
 
-CHUNK_SIZE = 170  # hex chars (100 bytes)
+CHUNK_SIZE = 170  # hex chars (85 tx bytes)
+
+# base64 spends 1.33 chars per transaction byte instead of hex's 2, so the same
+# 187-character message carries 126 tx bytes instead of 85. Kept a multiple of 4
+# so every chunk is whole base64 quanta. Override to calibrate for a radio or
+# region that leaves less room in the 233-byte Meshtastic payload.
+CHUNK_SIZE_B64 = int(os.environ.get("BTCMESH_CHUNK_SIZE_B64", "168"))
+assert (
+    CHUNK_SIZE_B64 > 0 and CHUNK_SIZE_B64 % 4 == 0
+), "BTCMESH_CHUNK_SIZE_B64 must be a positive multiple of 4"
 
 
 def chunk_transaction(tx_hex, chunk_size):
     """Split a hex string into chunks of specified size."""
     return [tx_hex[i : i + chunk_size] for i in range(0, len(tx_hex), chunk_size)]
 
 
+def encode_payload(tx_hex, encoding):
+    """Encode a raw transaction hex string for transport over the mesh."""
+    if encoding == ENCODING_B64:
+        return base64.b64encode(bytes.fromhex(tx_hex)).decode("ascii")
+    return tx_hex
+
+
 def generate_session_id():
     """Generate a unique 5-character hex session ID."""
     return uuid.uuid4().hex[:5]
@@ -207,6 +230,16 @@ def cli_main(
             action="store_true",
             help="Only parse and print arguments, do not send",
         )
+        parser.add_argument(
+            "--encoding",
+            choices=[ENCODING_HEX, ENCODING_B64],
+            default=ENCODING_HEX,
+            help=(
+                "Payload encoding. 'base64' (protocol v1.1) carries ~48%% more "
+                "transaction per message and needs fewer ACK round trips, but "
+                "the relay must understand BTC_TX64. Default: hex (v1.0)."
+            ),
+        )
         args = parser.parse_args()
     tx_hex = args.tx
     if len(tx_hex) % 2 != 0 or not re.fullmatch(r"[0-9a-fA-F]+", tx_hex):
@@ -216,15 +249,21 @@ def cli_main(
         )
         raise ValueError("Invalid raw transaction hex")
     logger = injected_logger if injected_logger is not None else cli_logger
+    # Callers that build their own args object (the GUI, tests) predate the
+    # flag, so default to the v1.0 encoding rather than requiring the attribute.
+    encoding = getattr(args, "encoding", None) or ENCODING_HEX
+    chunk_prefix = CHUNK_PREFIX_B64 if encoding == ENCODING_B64 else CHUNK_PREFIX
+    chunk_size = CHUNK_SIZE_B64 if encoding == ENCODING_B64 else CHUNK_SIZE
+    wire_payload = encode_payload(tx_hex, encoding)
     if args.dry_run:
         print(f"Arguments parsed successfully:")
         print(f"  Destination: {args.destination}")
         print(f"  Raw TX Hex: {args.tx}")
         session_id = getattr(args, "session_id", None) or generate_session_id()
-        chunks = chunk_transaction(tx_hex, CHUNK_SIZE)
+        chunks = chunk_transaction(wire_payload, chunk_size)
         total_chunks = len(chunks)
         for i, payload in enumerate(chunks, 1):
-            print(f"BTC_TX|{session_id}|{i}/{total_chunks}|{payload}")
+            print(f"{chunk_prefix}{session_id}|{i}/{total_chunks}|{payload}")
         return 0
     iface = (
         injected_iface
@@ -247,7 +286,7 @@ def cli_main(
     )  # Set global session ID
     _expected_sender_node_id = args.destination  # Set global expected sender
 
-    chunks = chunk_transaction(tx_hex, CHUNK_SIZE)
+    chunks = chunk_transaction(wire_payload, chunk_size)
     total_chunks = len(chunks)
 
     # Determine if we are using injected receiver or setting up our own
@@ -288,7 +327,7 @@ def cli_main(
                 current_retries = 0
 
                 while current_retries < MAX_RETRIES:
-                    msg_to_send = f"BTC_TX|{_current_session_id}|{chunk_num}/{total_chunks}|{payload}"
+                    msg_to_send = f"{chunk_prefix}{_current_session_id}|{chunk_num}/{total_chunks}|{payload}"
                     try:
                         iface.sendText(text=msg_to_send, destinationId=args.destination)
                         print(
@@ -607,6 +646,13 @@ def cli_main(
             logger.info(
                 f"SystemExit caught in cli_main for session {_current_session_id} with code {e.code}"
             )
+            if e.code and encoding == ENCODING_B64:
+                # A relay running protocol v1.0 ignores BTC_TX64 outright, so
+                # the symptom is silence rather than a NACK.
+                print(
+                    "No ACK from the relay. If it runs protocol v1.0 (hex only), "
+                    "retry with --encoding hex."
+                )
             return e.code
         except Exception as e:
             logger.error(

btcmesh_server.py

@@ -22,6 +22,8 @@
     ReassemblyError,
     CHUNK_PREFIX,
     CHUNK_PARTS_DELIMITER,
+    MismatchedEncodingError,
+    match_chunk_prefix,
 )
 from core.rpc_client import BitcoinRPCClient
 from core.transaction_history import TransactionHistory
@@ -82,8 +84,9 @@ def _format_node_id(node_id_val: Any) -> Optional[str]:
 def _extract_session_id_from_raw_chunk(message_text: str) -> Optional[str]:
     """Rudimentary attempt to extract tx_session_id for NACKs if full parsing fails."""
     try:
-        if message_text.startswith(CHUNK_PREFIX):
-            parts = message_text[len(CHUNK_PREFIX) :].split(CHUNK_PARTS_DELIMITER)
+        matched = match_chunk_prefix(message_text)
+        if matched is not None:
+            parts = matched[0].split(CHUNK_PARTS_DELIMITER)
             if len(parts) > 0 and parts[0]:
                 return parts[0]
     except Exception:  # pylint: disable=broad-except
@@ -223,13 +226,12 @@ def on_receive_text_message(
                 f"Direct text from {sender_node_id_for_reply}: '{message_text}'"
             )
 
-            if message_text.startswith(CHUNK_PREFIX):
+            matched_chunk = match_chunk_prefix(message_text)
+            if matched_chunk is not None:
                 # Log every received chunk for diagnostics
                 try:
                     # Always parse session_id and chunk_info for NACK, even if add_chunk fails
-                    parts = message_text[len(CHUNK_PREFIX) :].split(
-                        CHUNK_PARTS_DELIMITER
-                    )
+                    parts = matched_chunk[0].split(CHUNK_PARTS_DELIMITER)
                     session_id = parts[0] if len(parts) > 0 else "UNKNOWN"
                     chunk_info = parts[1] if len(parts) > 1 else "UNKNOWN"
                 except Exception:
@@ -351,7 +353,11 @@ def on_receive_text_message(
                                     error=str(error),
                                     raw_tx=reassembled_hex
                                 )
-                except (InvalidChunkFormatError, MismatchedTotalChunksError) as e:
+                except (
+                    InvalidChunkFormatError,
+                    MismatchedTotalChunksError,
+                    MismatchedEncodingError,
+                ) as e:
                     tx_session_id_for_nack = (
                         session_id
                         or _extract_session_id_from_raw_chunk(message_text)

core/reassembler.py

@@ -1,19 +1,46 @@
 from __future__ import annotations
 
+import base64
 import time
 from typing import Dict, Optional, Tuple, List, Any
 
 from core.logger_setup import server_logger  # Assuming a logger is available
 
 # Constants for chunk parsing
 CHUNK_PREFIX = "BTC_TX|"
+CHUNK_PREFIX_B64 = "BTC_TX64|"
 CHUNK_PARTS_DELIMITER = "|"
 CHUNK_INDEX_TOTAL_DELIMITER = "/"
 
+# Payload encodings. Hex is protocol v1.0; base64 (v1.1) carries ~48% more
+# transaction per message because it spends 1.33 chars per byte instead of 2.
+ENCODING_HEX = "hex"
+ENCODING_B64 = "base64"
+
 # Default timeout for reassembly sessions in seconds
 DEFAULT_REASSEMBLY_TIMEOUT_SECONDS = 5 * 60  # 5 minutes
 
 
+def match_chunk_prefix(message_text: str) -> Optional[Tuple[str, str]]:
+    """
+    Identify a transaction chunk message and its payload encoding.
+
+    Returns (remainder_after_prefix, encoding) or None if the message is not a
+    transaction chunk.
+
+    Note that "BTC_TX64|..." deliberately does not match CHUNK_PREFIX
+    ("BTC_TX|", with the pipe), so a v1.0 relay ignores base64 chunks outright
+    instead of misparsing them as hex.
+    """
+    for prefix, encoding in (
+        (CHUNK_PREFIX_B64, ENCODING_B64),
+        (CHUNK_PREFIX, ENCODING_HEX),
+    ):
+        if message_text.startswith(prefix):
+            return message_text[len(prefix) :], encoding
+    return None
+
+
 class ReassemblyError(Exception):
     """Custom exception for reassembly errors."""
 
@@ -32,6 +59,12 @@ class DuplicateChunkError(ReassemblyError):
     pass
 
 
+class MismatchedEncodingError(ReassemblyError):
+    """Raised when the payload encoding changes mid-session."""
+
+    pass
+
+
 class InvalidChunkFormatError(ReassemblyError):
     """Raised when a chunk format is invalid."""
 
@@ -70,25 +103,32 @@ def __init__(self, timeout_seconds: int = DEFAULT_REASSEMBLY_TIMEOUT_SECONDS):
             f"TransactionReassembler initialized with timeout: {timeout_seconds}s"
         )
 
-    def _parse_chunk(self, message_text: str) -> Tuple[str, int, int, str]:
+    def _parse_chunk(self, message_text: str) -> Tuple[str, int, int, str, str]:
         """
         Parses a raw message string to extract transaction chunk components.
 
-        Format: "BTC_TX|<tx_session_id>|<chunk_num>/<total_chunks>|<hex_payload_part>"
+        Formats:
+            "BTC_TX|<tx_session_id>|<chunk_num>/<total_chunks>|<hex_payload_part>"
+            "BTC_TX64|<tx_session_id>|<chunk_num>/<total_chunks>|<base64_payload_part>"
 
         Args:
             message_text: The raw text message received.
 
         Returns:
-            A tuple containing (tx_session_id, chunk_num, total_chunks, hex_payload_part).
+            A tuple containing
+            (tx_session_id, chunk_num, total_chunks, payload_part, encoding).
 
         Raises:
             InvalidChunkFormatError: If the message_text does not conform to the expected format.
         """
-        if not message_text.startswith(CHUNK_PREFIX):
-            raise InvalidChunkFormatError(f"Message does not start with {CHUNK_PREFIX}")
+        matched = match_chunk_prefix(message_text)
+        if matched is None:
+            raise InvalidChunkFormatError(
+                f"Message does not start with {CHUNK_PREFIX} or {CHUNK_PREFIX_B64}"
+            )
+        remainder, encoding = matched
 
-        parts = message_text[len(CHUNK_PREFIX) :].split(CHUNK_PARTS_DELIMITER)
+        parts = remainder.split(CHUNK_PARTS_DELIMITER)
         if len(parts) != 3:
             raise InvalidChunkFormatError(
                 f"Message does not have 3 parts after prefix: {parts}"
@@ -125,7 +165,7 @@ def _parse_chunk(self, message_text: str) -> Tuple[str, int, int, str]:
                 f"Invalid chunk numbering: {chunk_num}/{total_chunks}"
             )
 
-        return tx_session_id, chunk_num, total_chunks, hex_payload_part
+        return tx_session_id, chunk_num, total_chunks, hex_payload_part, encoding
 
     def add_chunk(self, sender_id: Any, message_text: str) -> Optional[str]:
         """
@@ -159,7 +199,7 @@ def add_chunk(self, sender_id: Any, message_text: str) -> Optional[str]:
         session_key = sender_id
 
         try:
-            tx_session_id, chunk_num, total_chunks, hex_payload_part = (
+            tx_session_id, chunk_num, total_chunks, hex_payload_part, encoding = (
                 self._parse_chunk(message_text)
             )
         except InvalidChunkFormatError as e:
@@ -184,12 +224,24 @@ def add_chunk(self, sender_id: Any, message_text: str) -> Optional[str]:
             sender_sessions[tx_session_id] = {
                 "chunks": {},
                 "total_chunks": total_chunks,
+                "encoding": encoding,
                 "last_update_time": current_time,
                 "sender_id_str": str(sender_id),  # Store original sender_id for replies
             }
 
         session_data = sender_sessions[tx_session_id]
 
+        # Check for consistency in payload encoding. Concatenating hex and
+        # base64 fragments would silently produce a corrupt transaction.
+        if session_data.get("encoding", ENCODING_HEX) != encoding:
+            error_msg = (
+                f"{log_ctx} Mismatched encoding. Expected "
+                f"{session_data.get('encoding')}, got {encoding}. Discarding session."
+            )
+            server_logger.error(error_msg)
+            del sender_sessions[tx_session_id]
+            raise MismatchedEncodingError(error_msg)
+
         # Check for consistency in total_chunks
         if session_data["total_chunks"] != total_chunks:
             error_msg = (
@@ -237,6 +289,22 @@ def add_chunk(self, sender_id: Any, message_text: str) -> Optional[str]:
 
                 reassembled_hex += session_data["chunks"][i]
 
+            # base64 sessions carry the transaction more densely on air; decode
+            # here so callers always receive hex and never learn about encoding.
+            if session_data.get("encoding") == ENCODING_B64:
+                try:
+                    reassembled_hex = base64.b64decode(
+                        reassembled_hex, validate=True
+                    ).hex()
+                except ValueError as e:  # binascii.Error subclasses ValueError
+                    error_msg = (
+                        f"{log_ctx} Reassembly failed: payload is not valid "
+                        f"base64 ({e})."
+                    )
+                    server_logger.error(error_msg)
+                    del sender_sessions[tx_session_id]
+                    raise InvalidChunkFormatError(error_msg)
+
             server_logger.info(f"{log_ctx} Reassembly successful.")
             # Clean up completed session
             del sender_sessions[tx_session_id]