retrace

Author: Jemoka

batchalign/pipelines/cleanup/retrace.py

@@ -22,7 +22,7 @@ def process(self, doc, **kwargs):
                 if i.type in [TokenType.REGULAR, TokenType.PUNCT, TokenType.FP]:
                     content.append(i)
             # scan for n-gram retraces
-            for n in range(1, len(content)):
+            for n in range(1 if "yue" not in doc.langs and "zho" not in doc.langs else 2, len(content)):
                 begin = 0
                 while begin < len(content)-(n):
                     # get the n gram info; we convert it to

batchalign/version

@@ -1,3 +1,3 @@
-0.7.19-post.5
+0.7.19-post.6
 May 23th, 2025
 abbreviations