Update (dev) fixed download sort of

aligned-blocks.csv

@@ -0,0 +1,12 @@
+text1,text2,llh
+Wuthering Heights,��Хɽׯ,0.45
+--------------------------------------------------------------------------------,--------------------------------------------------------------------------------,1.0
+Chapter 2,�ڶ���,0.91
+Chinese,Ӣ��,0.52
+Yesterday afternoon set in misty and cold.,��������������������,0.74
+"I had half a mind to spend it by my study fire, instead of wading through heath and mud to Wuthering Heights.",��������鷿¯����ĥһ���磬��������Ӳ����ൽ��Хɽׯ�ˡ�,0.45
+"On coming up from dinner, however (N.B. I dine between twelve and one o'clock; the housekeeper, a matronly lady, taken as a fixture along with the house, could not, or would not, comprehend my request that I might be served at five), on mounting the stairs with this lazy intention, and stepping into the room, I saw a servant girl on her knees surrounded by brushes and coal-scuttles, and raising an infernal dust as she extinguished the flames with heaps of cinders.",���ǣ��Թ��緹��ע�⡪������ʮ������һ����֮����緹�������Ե����������ӵĸ�����Ĺܼ��ţ�һλ�����̫̫ȴ���ܣ����߲���Ը����������������ӿ��������⣩�����һ������������뷨����¥���������ӵ�ʱ�򣬿���һ��Ů�͹��ڵ��ϣ�������ɨ���ú����,0.72
+,��������һ�Ѷ�ú����𣬸���һƬ�����Ļҳ���,
+,�⾰�����̰��Ҹϻ�ͷ�ˡ�,
+"This spectacle drove me back immediately; I took my hat, and, after a four-miles' walk, arrived at Heathcliff's garden gate just in time to escape the first feathery flakes of a snow shower.",������ñ�ӣ���������·��������ϣ�̿�����Ļ�԰�ڿڣ��պö����һ����������Ķ�ë��ѩ��,0.68
+"On that bleak hill top the earth was hard with a black frost, and the air made me shiver through every limb.",���ǻ�����ɽ���ϣ�������,0.43

app.py

@@ -87,9 +87,19 @@ def ml_fn(
             logger.exception(exc)
             aset = [["", "", ""]]
 
-    _ = len(paras1) + len(paras2)
-    av = f"{t.duration / _ * 1000:.2f}"
-    logger.info(" %s blocks, took %s, av. %s s/1000 blk", _, t.duration_human, av)
+    len1 = len(paras1)
+    len2 = len(paras2)
+    ic(len1, len2)
+
+    if not (len1 and len2):
+        _ = "At least one text is empty... nothing to do."
+        return pd.DataFrame([[_]]), None, None
+
+    av = ""
+    len12 = len1 + len2
+    if len12:
+        av = f"{t.duration / len12 * 1000:.2f}"
+    logger.info(" %s blocks, took %s, av. %s s/1000 blk", len12, t.duration_human, av)
 
     pairs = aset2pairs(paras1, paras2, aset)
     df = pd.DataFrame(pairs, columns=["text1", "text2", "llh"])
@@ -104,15 +114,15 @@ def ml_fn(
         try:
             dl_csv = Path("aligned-blocks.csv")
             csv_str = df.to_csv(index=False)
-            # dl_csv.write_text(csv_str, encoding="utf8")
+            dl_csv.write_text(csv_str, encoding="gbk")
             ic("Saving df.to_csv to dl_csv...")
         except Exception as exc:
             logger.exception(exc)
 
-    return df, html, csv_str
+    return df, html, dl_csv
 
 
-mlbee = gr.Interface(
+iface = gr.Interface(
     fn=ml_fn,
     inputs=[
         "textarea",
@@ -135,7 +145,7 @@ mlbee = gr.Interface(
     ],
 )
 
-mlbee.launch(
+iface.launch(
     show_error=True,
     enable_queue=True,
 )

install-sw.sh

@@ -1,24 +1,10 @@
-# pip install pipx
-# pipx install poetry
-# pipx ensurepath
-# source ~/.bashrc
-
-# curl -sSL https://install.python-poetry.org | python3 -
-# -C- continue -S show error -o output
 curl -sSL -C- -o install-poetry.py  https://install.python-poetry.org
 python install-poetry.py
 rm install-poetry.py
 echo export PATH=~/.local/bin:$PATH > ~/.bashrc
 source ~/.bashrc
-# ~/.local/bin/poetry install
-
-wget -c https://deb.nodesource.com/setup_12.x
-bash setup_12.x
-apt-get install -y nodejs
-npm install -g npm@latest
-npm install -g nodemon
-rm setup_12.x
 
-# apt upate  # alerady done in apt-get install -y nodejs
+apt update > /dev/null 2>&1
 apt install byobu -y > /dev/null 2>&1
-
+byobu-enable
+byobu

install-sw1.sh

@@ -0,0 +1,25 @@
+# pip install pipx
+# pipx install poetry
+# pipx ensurepath
+# source ~/.bashrc
+
+# curl -sSL https://install.python-poetry.org | python3 -
+# -C- continue -S show error -o output
+curl -sSL -C- -o install-poetry.py  https://install.python-poetry.org
+python install-poetry.py
+rm install-poetry.py
+echo export PATH=~/.local/bin:$PATH > ~/.bashrc
+source ~/.bashrc
+# ~/.local/bin/poetry install
+
+wget -c https://deb.nodesource.com/setup_12.x
+bash setup_12.x
+apt-get install -y nodejs
+npm install -g npm@latest
+npm install -g nodemon
+rm setup_12.x
+
+# apt update  # alerady done in apt-get install -y nodejs
+apt install byobu -y > /dev/null 2>&1
+byobu-enable
+byobu

poetry.lock

@@ -1517,7 +1517,7 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8.3"
-content-hash = "8a3e87bc76faae5ea5b48c1e0cdb5bf98a1b671e2d90baa30696f228d8a6113b"
+content-hash = "4d19a2267a01e6f89ec45ad178fe2bc0e9c4c36c07c1737be1e965480bcee049"
 
 [metadata.files]
 about-time = [

pyproject.toml

@@ -26,6 +26,7 @@ more-itertools = "^8.13.0"
 cchardet = "^2.1.7"
 typer = "^0.4.1"
 seg-text = "^0.1.2"
+joblib = "^1.1.0"
 
 [tool.poe.executor]
 type = "poetry"

radio_mlbee/gen_cmat.py

@@ -1,25 +1,38 @@
 """Gen cmat for de/en text."""
 # pylint: disable=invalid-name, too-many-branches
-
+import os
+from pathlib import Path
 from typing import List, Optional
 
 import more_itertools as mit
 import numpy as np
+import logzero
 
-# from model_pool import load_model_s
 from hf_model_s_cpu import model_s
+from joblib import Memory
 from logzero import logger
+from set_loglevel import set_loglevel
 from tqdm import tqdm
 
 from radio_mlbee.cos_matrix2 import cos_matrix2
 
+os.environ["LOGLEVEL"] = "10"  # turn debug on
+logzero.loglevel(set_loglevel())
+logger.debug(" debug is on ")
+
 try:
     model = model_s()
 except Exception as _:
     logger.exception(_)
     raise
 
+cachedir = Path("~").expanduser() / "cachedir"
+memory = Memory(cachedir, verbose=0)
+if set_loglevel() <= 10:
+    memory.clear()
+
 
+@memory.cache
 def gen_cmat(text1: List[str], text2: List[str], bsize: int = 50) -> np.ndarray:
     """Gen corr matrix for texts.