ForcePilot/web/src/utils/chunkUtils.js

/**
 * Chunk合并工具函数
 * 用于合并多个chunk并处理重叠内容
 */

/**
 * 查找两个字符串的重叠部分
 * @param {string} str1 - 第一个字符串
 * @param {string} str2 - 第二个字符串
 * @returns {string} - 重叠部分的内容
 */
export function findOverlap(str1, str2) {
  if (!str1 || !str2) return '';

  const maxOverlap = Math.min(str1.length, str2.length);
  let overlap = '';

  // 从最长可能的重叠开始检查
  for (let i = maxOverlap; i > 0; i--) {
    const endStr1 = str1.slice(-i);
    const startStr2 = str2.slice(0, i);

    if (endStr1 === startStr2) {
      overlap = endStr1;
      break;
    }
  }

  return overlap;
}

/**
 * 合并chunks并处理重叠内容
 * @param {Array} chunks - chunk数组，每个chunk包含id, content, chunk_order_index
 * @returns {Object} - 合并结果，包含content和chunks数组
 */
export function mergeChunks(chunks) {
  if (!chunks || chunks.length === 0) {
    return { content: '', chunks: [] };
  }

  // 按order排序
  const sorted = [...chunks].sort((a, b) => a.chunk_order_index - b.chunk_order_index);
  const merged = [];
  let currentContent = '';

  for (let i = 0; i < sorted.length; i++) {
    const chunk = sorted[i];
    const content = chunk.content;

    if (i === 0) {
      // 第一个chunk直接添加
      currentContent = content;
      merged.push({
        ...chunk,
        startOffset: 0,
        endOffset: content.length
      });
    } else {
      // 查找重叠部分
      const overlap = findOverlap(currentContent, content);
      const newContent = content.slice(overlap.length);

      if (newContent.length > 0) {
        const startOffset = currentContent.length;
        currentContent += newContent;
        merged.push({
          ...chunk,
          startOffset,
          endOffset: currentContent.length
        });
      }
    }
  }

  return { content: currentContent, chunks: merged };
}

/**
 * 将文本分割成段落
 * @param {string} content - 文本内容
 * @returns {Array} - 段落数组
 */
export function splitIntoParagraphs(content) {
  if (!content) return [];

  // 按换行符分割，保留空段落
  return content.split(/\n\n+/).filter(para => para.trim() !== '');
}

/**
 * 为每个段落找到对应的chunk
 * @param {Array} paragraphs - 段落数组
 * @param {Array} mappedChunks - 映射后的chunks
 * @returns {Array} - 包含chunk信息的段落
 */
export function mapParagraphsToChunks(paragraphs, mappedChunks) {
  if (!paragraphs || !mappedChunks) return [];

  let currentOffset = 0;
  return paragraphs.map(paragraph => {
    const paragraphLength = paragraph.length + 2; // +2 for the \n\n

    // 找到包含此位置的chunk
    const chunk = mappedChunks.find(chunk =>
      currentOffset >= chunk.startOffset && currentOffset < chunk.endOffset
    ) || mappedChunks[0];

    const result = {
      content: paragraph,
      chunk,
      startOffset: currentOffset,
      endOffset: currentOffset + paragraphLength
    };

    currentOffset += paragraphLength;
    return result;
  });
}

/**
 * 获取chunk的预览文本
 * @param {string} content - chunk内容
 * @param {number} maxLength - 最大长度
 * @returns {string} - 预览文本
 */
export function getChunkPreview(content, maxLength = 100) {
  if (!content) return '';

  const text = content.replace(/\n+/g, ' ').trim();
  if (text.length <= maxLength) return text;

  return text.slice(0, maxLength) + '...';
}