support IOStream

Author: guoyongzhi

Project.toml

@@ -1,7 +1,7 @@
 name = "LanguageIdentification"
 uuid = "35248bf2-58a5-46a0-9b1d-c0a73a50a105"
 authors = ["guoyongzhi <[email protected]>"]
-version = "1.0.0"
+version = "1.0.1"
 
 [compat]
 julia = "1.9"

src/ngrams.jl

@@ -39,7 +39,9 @@ function count_all_ngrams(text_list, n, counter = Dict{Vector{UInt8},Float32}();
     end
     counter
 end
-
+function count_all_ngrams(textfile::IO, args...; kwargs...)
+    count_all_ngrams(eachline(textfile), args...; kwargs...)
+end
 # function count_dataset_ngrams(dataset, n; kwargs...)
 #     counters = [Dict{Vector{UInt8},Float32}() for i in 1:n]
 #     for (text, lang) in dataset