11种Spring AI文档切割策略全解析

<dependencies>
    <!-- spring ai openai -->
    <dependency>
        <groupid>org.springframework.ai</groupid>
        <artifactid>spring-ai-starter-model-openai</artifactid>
    </dependency>
    <!-- spring ai elasticsearch vector store -->
    <dependency>
        <groupid>org.springframework.ai</groupid>
        <artifactid>spring-ai-starter-vector-store-elasticsearch</artifactid>
    </dependency>
    <!-- webflux -->
    <dependency>
        <groupid>org.springframework.boot</groupid>
        <artifactid>spring-boot-starter-webflux</artifactid>
    </dependency>
</dependencies>

# application.yml
spring:
  ai:
    openai:
      api-key: ${openai_api_key}
      embedding:
        options:
          model: text-embedding-3-small
    vectorstore:
      elasticsearch:
        index-name: document-store
        dimensions: 1536

/**
 * agentic 切割核心逻辑
 * 
 * 1. 首先使用递归切割获得初步块
 * 2. 识别主题转换标记词
 * 3. 在主题边界处优化切割点
 */
public list<document> splitagentic(string text, string filename, splitoptions options) {
    // 第一步：递归切割获得初步块
    list<textchunk> initialchunks = recursivesplitinternal(
        normalizetext(text),
        0,
        chunksize * 2, // 更大的初始块
        chunkoverlap,
        0
    );
    
    // 第二步：使用启发式规则优化切割点
    for (textchunk chunk : initialchunks) {
        // 识别主题转换标记词
        string optimizedcontent = optimizechunkboundary(content, chunksize);
        // ...
    }
}

// 主题转换标记词（中英文）
string[] topicmarkers = {
    // 标点符号
    "\n\n", "\n", ". ", "? ", "! ", "。", "？", "！",
    // 中文序列词
    "首先", "其次", "然后", "接下来", "最后",
    "第一", "第二", "第三", "另一方面", "此外", "总之"
};

# 上传长文档，使用 agentic 切割
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=agentic&chunksize=1500" \
  -f "file=@long-article.txt"

{
  "split_method": "agentic",
  "original_separator": "\n\n",
  "optimization_applied": true,
  "chunk_group": 0
}

/**
 * 代码感知切割
 * 
 * 1. 检测编程语言
 * 2. 解析代码块（类、方法等）
 * 3. 提取 import 语句
 * 4. 按代码结构切割
 */
public list<document> splitcodeaware(string text, string filename, splitoptions options) {
    // 检测语言
    string language = detectlanguage(filename);
    
    // 解析代码块
    list<codeblock> codeblocks = parsecodeblocks(text, language);
    
    // 提取 imports
    list<string> imports = extractimports(text, language);
    
    // 构建文档
    for (codeblock block : codeblocks) {
        map<string, object> metadata = buildcodemetadata(
            filename, chunkindex, block, imports, language
        );
        // ...
    }
}

// java 代码块解析正则
pattern pattern = pattern.compile(
    "(?:(public|private|protected|static|final|abstract|class|interface|enum|record|void|@\\w+)\\s+)?" +
    "(?:<[^>]+>\\s*)?" +
    "(\\w+)\\s*\\([^)]*\\)\\s*(?:throws\\s+\\w+(?:\\s*,\\s*\\w+)*)?\\s*\\{[^}]*\\}" +
    "|(?:public|private|protected)?\\s*(?:static\\s+)?(?:final\\s+)?(?:abstract\\s+)?" +
    "(?:class|interface|enum|record)\\s+(\\w+)",
    pattern.dotall
);

# 上传 java 文件
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=code_aware" \
  -f "file=@userservice.java"
# 上传 python 文件
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=code_aware" \
  -f "file=@main.py"

{
  "split_method": "code_aware",
  "language": "java",
  "block_type": "method",
  "block_name": "getuserbyid",
  "start_line": 45,
  "imports_count": 12,
  "has_imports": true
}

/**
 * 层次结构切割
 * 
 * 构建文档树：文档 → 章节 → 小节 → 段落
 */
public list<document> splithierarchical(string text, string filename, splitoptions options) {
    // 构建层次树
    documentnode root = buildhierarchicaltree(text, filename);
    
    // 展平树为文档列表
    list<document> documents = new arraylist<>();
    flattentree(root, documents, filename, 0, maxchunksize);
    
    return documents;
}

/**
 * 构建层次树
 */
private documentnode buildhierarchicaltree(string text, string filename) {
    documentnode root = new documentnode("root", "document", text, 0, 0);
    
    // 第一层：按章节分割
    string[] chapters = text.split(
        "(?=(?:^|\\n)\\s*(?:第[一二三四五六七八九十\\d]+[章节篇]|chapter\\s+\\d+|\\d+\\.\\s+[^\\n]+))"
    );
    
    // 第二层：按小节分割
    // 第三层：按段落分割
    // ...
}

# 上传结构化长文档
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=hierarchical&chunksize=2000" \
  -f "file=@book.txt"

{
  "split_method": "hierarchical",
  "node_id": "section_0_1",
  "node_type": "section",
  "hierarchy_level": 2,
  "node_index": 1,
  "has_children": true,
  "children_count": 3
}

/**
 * 滑动窗口切割
 * 
 * 特点：
 * - 高重叠率（默认 50%）
 * - 确保没有信息遗漏
 * - 适合检索增强生成 (rag)
 */
public list<document> splitslidingwindow(string text, string filename, splitoptions options) {
    int windowsize = options.getchunksize() != null ? options.getchunksize() : default_chunk_size;
    int overlapratio = options.getchunkoverlap() != null ? options.getchunkoverlap() : 50;
    
    // 计算步长
    int stride = math.max(1, windowsize * (100 - overlapratio) / 100);
    
    while (startindex < normalizedtext.length()) {
        int endindex = math.min(startindex + windowsize, normalizedtext.length());
        
        // 尝试在句子边界结束
        if (endindex < normalizedtext.length()) {
            endindex = findbestsplitpoint(normalizedtext, endindex);
        }
        
        // 滑动窗口
        startindex += stride;
    }
}

# 高重叠率（70%）- 适合高精度检索
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=sliding_window&chunksize=1000&chunkoverlap=70" \
  -f "file=@document.txt"
# 中等重叠率（50%）- 平衡方案
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=sliding_window&chunksize=1000&chunkoverlap=50" \
  -f "file=@document.txt"

{
  "split_method": "sliding_window",
  "window_start": 0,
  "window_end": 1000,
  "window_size": 1000,
  "stride": 500,
  "overlap_ratio": 50,
  "overlap_with_previous": 0
}

/**
 * 检测结构化数据格式
 */
private string detectstructuredformat(string filename, string content) {
    string lower = filename.tolowercase();
    if (lower.endswith(".csv")) return "csv";
    if (lower.endswith(".json")) return "json";
    if (lower.endswith(".xml")) return "xml";
    if (lower.endswith(".yaml") || lower.endswith(".yml")) return "yaml";
    
    // 根据内容检测
    string trimmed = content.trim();
    if (trimmed.startswith("[") || trimmed.startswith("{")) return "json";
    if (trimmed.startswith("<?xml") || trimmed.startswith("<")) return "xml";
    // ...
}

/**
 * csv 切割 - 保留表头
 */
private list<document> splitcsv(string text, string filename, splitoptions options) {
    int maxrows = options.getchunksize() != null ? options.getchunksize() / 50 : 100;
    
    // 提取表头
    string header = lines[0];
    list<string> headers = parsecsvline(header);
    
    // 每个块都包含表头
    for (string line : datalines) {
        if (rowcount >= maxrows) {
            // 保存当前块
            // 新块以表头开始
            currentchunk.append(header).append("\n");
        }
        currentchunk.append(line).append("\n");
    }
}

/**
 * json 切割 - 按对象/条目分割
 */
private list<document> splitjson(string text, string filename, splitoptions options) {
    // json 数组：提取每个对象
    if (trimmed.startswith("[")) {
        list<string> objects = extractjsonobjects(trimmed);
        processjsonobjects(objects, filename, documents, maxsize);
    }
    // json 对象：按顶层键分割
    else if (trimmed.startswith("{")) {
        list<string> entries = extractjsonentries(trimmed);
        processjsonobjects(entries, filename, documents, maxsize);
    }
}

/**
 * xml 切割 - 按元素分割
 */
private list<document> splitxml(string text, string filename, splitoptions options) {
    // 提取 xml 声明
    string declaration = "";
    int declend = text.indexof("?>");
    if (declend > 0) {
        declaration = text.substring(0, declend + 2) + "\n";
    }
    
    // 提取主要元素
    list<string> elements = extractxmlelements(text);
    
    // 每个块包含声明
    for (string elem : elements) {
        currentchunk.append(declaration);
        currentchunk.append(elem);
    }
}

# csv 文件
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=structured_data" \
  -f "file=@users.csv"
# json 文件
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=structured_data" \
  -f "file=@data.json"
# xml 文件
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=structured_data" \
  -f "file=@config.xml"
# yaml 文件
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=structured_data" \
  -f "file=@config.yaml"

{
  "split_method": "structured_data",
  "format": "csv",
  "start_index": 1,
  "end_index": 100,
  "record_count": 100,
  "headers": ["id", "name", "email"],
  "header_count": 3
}

{
  "split_method": "structured_data",
  "format": "json",
  "start_index": 0,
  "end_index": 49,
  "record_count": 50
}

# 基础策略
post /api/vector-store/upload?splitstrategy=recursive
post /api/vector-store/upload?splitstrategy=markdown
post /api/vector-store/upload?splitstrategy=token
post /api/vector-store/upload?splitstrategy=semantic
post /api/vector-store/upload?splitstrategy=smart_paragraph
post /api/vector-store/upload?splitstrategy=character
# 新策略
post /api/vector-store/upload?splitstrategy=agentic
post /api/vector-store/upload?splitstrategy=code_aware
post /api/vector-store/upload?splitstrategy=hierarchical
post /api/vector-store/upload?splitstrategy=sliding_window
post /api/vector-store/upload?splitstrategy=structured_data

# 1. agentic 切割 - 长文章
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=agentic&chunksize=1500&chunkoverlap=200" \
  -h "content-type: multipart/form-data" \
  -f "file=@article.txt"
# 2. 代码感知切割 - java 文件
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=code_aware" \
  -f "file=@userservice.java"
# 3. 层次结构切割 - 书籍
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=hierarchical&chunksize=3000" \
  -f "file=@book.txt"
# 4. 滑动窗口切割 - rag 场景
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=sliding_window&chunksize=1000&chunkoverlap=70" \
  -f "file=@knowledge-base.txt"
# 5. 结构化数据切割 - csv
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=structured_data" \
  -f "file=@products.csv"
# 6. 结构化数据切割 - json
curl -x post "http://localhost:8080/api/vector-store/upload?splitstrategy=structured_data" \
  -f "file=@api-docs.json"

{
  "success": true,
  "message": "file uploaded and processed successfully",
  "data": {
    "filename": "article.txt",
    "chunkcount": 12,
    "totalcharacters": 15420,
    "splitmethod": "agentic",
    "message": "processed with agentic strategy"
  }
}

// 对于大文件，建议：
// 1. 使用流式处理
// 2. 增加 chunksize 减少块数
// 3. 使用异步处理

// 推荐配置
splitoptions options = new splitoptions()
    .withchunksize(2000)      // 更大的块
    .withchunkoverlap(100);   // 较小的重叠

// 批量上传多个文件
for (file file : files) {
    // 根据文件类型选择策略
    splitstrategy strategy = selectstrategy(file);
    
    documentsplitterservice.split(
        readfile(file),
        file.getname(),
        strategy,
        options
    );
}

// 对于重复处理的文档，可以缓存切割结果
@cacheable(value = "documentchunks", key = "#filename + '_' + #strategy")
public list<document> split(string text, string filename, splitstrategy strategy) {
    // ...
}