load a url

Author: Jemoka

batchalign/cli/cli.py

@@ -154,6 +154,9 @@ def writer(doc, output):
               default=False, help="Perform speaker diarization (this flag is ignored with Rev.AI)")
 @click.option("--wor/--nowor",
               default=False, help="Should we write word level alignment line? Default to no.")
+@click.option("--data",
+              help="the URL of the data",
+              type=str)
 @click.option("--lang",
               help="sample language in three-letter ISO 3166-1 alpha-3 code",
               show_default=True,

batchalign/cli/dispatch.py

@@ -5,6 +5,7 @@
 """
 
 from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn, BarColumn
+from urllib.parse import urlparse
 
 import warnings
 
@@ -63,6 +64,15 @@ def _dispatch(command, lang, num_speakers,
     files = []
     outputs = []
 
+    if kwargs.get("data"):
+        url = kwargs.get("data")
+        url = urlparse(url)
+        if url.scheme == "":
+            url = url._replace(scheme="http")
+        base = os.path.basename(url.path)
+        files.append(url)
+        outputs.append(os.path.join(out_dir, base))
+
     for basedir, _, fs in os.walk(in_dir):
         for f in fs:
             path = Path(os.path.join(basedir, f))
@@ -128,7 +138,8 @@ def _dispatch(command, lang, num_speakers,
         errors = []
         # create the spinner bars
         for f in files:
-            tasks[f] = prog.add_task(Path(f).name, start=False, processor="")
+            tasks[f] = prog.add_task(Path(f).name if isinstance(f, str) else Path(f.geturl()).name,
+                                     start=False, processor="")
 
         # create pipeline and read files
         baL.debug("Attempting to create BatchalignPipeline for CLI...")
@@ -152,7 +163,7 @@ def progress_callback(file, step, total, tools):
                 prog.start_task(tasks[file])
                 with warnings.catch_warnings(record=True) as w:
                     # parse the input format, as needed
-                    doc = loader(os.path.abspath(file))
+                    doc = loader(os.path.abspath(file) if isinstance(file, str) else file.geturl())
                     # if we ended up with a tuple of length two,
                     # that means that the loader requested kwargs
                     kw = {}
@@ -179,7 +190,7 @@ def progress_callback(file, step, total, tools):
     if len(errors) > 0:
         C.print()
         for file, trcbk, e in errors:
-            C.print(f"[bold red]ERROR[/bold red] on file [italic]{os.path.relpath(str(Path(file).absolute()), in_dir)}[/italic]: {escape(str(e))}\n")
+            C.print(f"[bold red]ERROR[/bold red] on file [italic]{os.path.relpath(str(Path(file).absolute()), in_dir) if isinstance(file, str) else file.geturl()}[/italic]: {escape(str(e))}\n")
             if ctx.obj["verbose"] == 1:
                 C.print(escape(str(trcbk)))
             elif ctx.obj["verbose"] > 1:

batchalign/pipelines/asr/tencent.py

@@ -74,16 +74,21 @@ def generate(self, f, **kwargs):
 
         L.info(f"Uploading '{pathlib.Path(f).stem}'...")
         # we will send the file for processing
-        with open(f, "rb") as image_file:
-            encoded_string = base64.b64encode(image_file.read())
+        if not str(f).startswith("http"):
+            with open(f, "rb") as image_file:
+                encoded_string = base64.b64encode(image_file.read())
 
         req = models.CreateRecTaskRequest()
         req.EngineModelType = f"16k_{lang}"
         req.ResTextFormat = 1
-        req.SourceType = 1
         req.SpeakerDiarization = 1
         req.ChannelNum = 1
-        req.Data = encoded_string.decode('ascii')
+        if not str(f).startswith("http"):
+            req.Data = encoded_string.decode('ascii')
+            req.SourceType = 1
+        else:
+            req.Url = f
+            req.SourceType = 0
         resp = client.CreateRecTask(req)
 
         L.info(f"Tencent is transcribing '{pathlib.Path(f).stem}'...")
@@ -96,7 +101,7 @@ def generate(self, f, **kwargs):
             res = client.DescribeTaskStatus(req)
 
         # if failed, raise
-        if res.Data.Status == "3":
+        if res.Data.Status == "3" or res.Data.Status == 3:
             raise RuntimeError(f"Tencent reports job failed! error='{res.Data.ErrorMsg}'")
 
         turns = []

batchalign/version

@@ -1,3 +1,3 @@
-0.7.17-post.15
+0.7.17-post.16
 March 26th, 2025
-better coref model
+better tencent ASR