feat: split text support multiple languages #593

2024-05-23 22:27:58 +08:00
parent bcf311ff19
commit 1cb07af75e
1 changed files with 21 additions and 13 deletions
@@ -107,19 +107,27 @@ export function getUUID(): string {

 export function splitText(text: string): string[] {
  const data: string[] = []
-  const symbolMap = new Map<number, string>()
-  for (const match of text.matchAll(UNICODE_SYMBOL_REG)) {
-    symbolMap.set(match.index!, match[0])
-  }
-  let t = 0
-  while (t < text.length) {
-    const symbol = symbolMap.get(t)
-    if (symbol) {
-      data.push(symbol)
-      t += symbol.length
-    } else {
-      data.push(text[t])
-      t++
+  if (Intl.Segmenter) {
+    const segmenter = new Intl.Segmenter()
+    const segments = segmenter.segment(text)
+    for (const { segment } of segments) {
+      data.push(segment)
+    }
+  } else {
+    const symbolMap = new Map<number, string>()
+    for (const match of text.matchAll(UNICODE_SYMBOL_REG)) {
+      symbolMap.set(match.index!, match[0])
+    }
+    let t = 0
+    while (t < text.length) {
+      const symbol = symbolMap.get(t)
+      if (symbol) {
+        data.push(symbol)
+        t += symbol.length
+      } else {
+        data.push(text[t])
+        t++
+      }
    }
  }
  return data