Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Agent_module/carbon_data/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from .embedders import BaseEmbedder, Embedder, LambdaEmbedder, NullEmbedder
from .models import (
Chunk,
Embedding,
Entity,
MemoryHit,
MemoryItem,
Expand Down Expand Up @@ -55,7 +54,6 @@ def create(path: PathLike, *, exist_ok: bool = False) -> CarbonStore:
"CarbonStore",
"Entity",
"Chunk",
"Embedding",
"Relation",
"MemoryItem",
"MemoryHit",
Expand Down
27 changes: 11 additions & 16 deletions Agent_module/carbon_data/chunkers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,8 @@ def _chunk(text: str) -> list[str]:
return _chunk


def by_paragraph(*, min_chars: int = 0) -> Chunker:
"""
Split on runs of blank lines. Drops any chunk shorter than
``min_chars`` after stripping; set to 0 to keep every paragraph.
"""
splitter = re.compile(r"\n\s*\n")
def _regex_chunker(splitter: "re.Pattern[str]", min_chars: int) -> Chunker:
"""Split on ``splitter``; keep stripped pieces of at least ``min_chars``."""

def _chunk(text: str) -> list[str]:
return [
Expand All @@ -73,22 +69,21 @@ def _chunk(text: str) -> list[str]:
return _chunk


def by_paragraph(*, min_chars: int = 0) -> Chunker:
"""
Split on runs of blank lines. Drops any chunk shorter than
``min_chars`` after stripping; set to 0 to keep every paragraph.
"""
return _regex_chunker(re.compile(r"\n\s*\n"), min_chars)


def by_sentence(*, min_chars: int = 0) -> Chunker:
"""
Naive sentence splitter: end-of-sentence punctuation followed by
whitespace and a capital letter. Good enough for prose; for legal/
technical text use ``by_tokens`` or a real tokenizer.
"""
splitter = re.compile(r"(?<=[.!?])\s+(?=[A-Z])")

def _chunk(text: str) -> list[str]:
return [
s.strip()
for s in splitter.split(text)
if s.strip() and len(s.strip()) >= min_chars
]

return _chunk
return _regex_chunker(re.compile(r"(?<=[.!?])\s+(?=[A-Z])"), min_chars)


__all__ = ["Chunker", "by_tokens", "by_paragraph", "by_sentence"]
19 changes: 1 addition & 18 deletions Agent_module/carbon_data/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,8 @@
from __future__ import annotations

import time
import uuid
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Optional

if TYPE_CHECKING:
import numpy as np


def _uuid() -> str:
return str(uuid.uuid4())
from typing import Any, Optional


@dataclass
Expand All @@ -39,14 +31,6 @@ class Chunk:
token_count: Optional[int] = None


@dataclass
class Embedding:
chunk_id: str
model: str
dim: int
vector: "np.ndarray"


@dataclass
class Relation:
src: str
Expand Down Expand Up @@ -121,7 +105,6 @@ class ValidationReport:
__all__ = [
"Entity",
"Chunk",
"Embedding",
"Relation",
"MemoryItem",
"SearchHit",
Expand Down
Loading
Loading