v0.4.13

Author: hscspring

README.md

@@ -498,6 +498,8 @@ Length of all numbers:  3
 
 ### 魔术方法
 
+#### 字典
+
 ```python
 from pnlp import MagicDict
 
@@ -522,6 +524,14 @@ print(pmag.MagicDict.reverse(dx))
 """
 ```
 
+#### 获取唯一文件名
+
+```python
+from pnlp import get_unique_fn
+
+get_unique_fn("a/b/c.md") == "a_b_c.md"
+```
+
 ### 并行处理
 
 支持四种并行处理方式：
@@ -561,6 +571,17 @@ def get_primes(lst):
 
 `concurring` 装饰器让你的迭代函数并行。
 
+### 后台处理
+
+```python
+from pnlp import run_in_new_thread
+
+def func(file, a, b, c):
+    background_task()
+
+run_in_new_thread(func, file, 1, 2, 3)
+```
+
 ## 测试
 
 Clone 仓库后执行：

README_EN.md

@@ -493,6 +493,8 @@ Length of all numbers:  3
 
 ### Magic
 
+#### MagicDict
+
 ```python
 from pnlp import MagicDict
 
@@ -517,6 +519,14 @@ print(pmag.MagicDict.reverse(dx))
 """
 ```
 
+#### GetUniqueFileName
+
+```python
+from pnlp import get_unique_fn
+
+get_unique_fn("a/b/c.md") == "a_b_c.md"
+```
+
 ### Concurring
 
 Support 4 types of concurring:
@@ -556,6 +566,17 @@ def get_primes(lst):
 
 `concurring` wrapper just make your original function concurring.  
 
+### Background
+
+```python
+from pnlp import run_in_new_thread
+
+def func(file, a, b, c):
+    background_task()
+
+run_in_new_thread(func, file, 1, 2, 3)
+```
+
 ## Test
 
 Clone the repo run:
@@ -566,6 +587,10 @@ $ python -m pytest
 
 ## ChangeLog
 
+**v0.4.13**
+
+Feat: Background task and magic get unique file name from file path.
+
 **v0.4.12**
 
 Feat: Subsentence cut and combine with a given threshold.

pnlp/__init__.py

@@ -8,11 +8,11 @@
 from pnlp.pnorm import NumNorm
 from pnlp.penh import TokenLevelSampler, SentenceLevelSampler
 from pnlp.ptrans import pick_entity_from_bio_labels, generate_uuid
-from pnlp.pmag import MagicDict
+from pnlp.pmag import MagicDict, get_unique_fn
 from pnlp.stopwords import StopWords
 from pnlp.stopwords import chinese_stopwords, english_stopwords
 
-from pnlp.utils import pstr, concurring, divide2int
+from pnlp.utils import pstr, concurring, divide2int, run_in_new_thread
 from pnlp.utils import generate_batches_by_num, generate_batches_by_size
 
 
@@ -24,7 +24,7 @@
 
 
 __title__ = "pnlp"
-__version__ = "0.4.12"
+__version__ = "0.4.13"
 __author__ = "Yam"
 __license__ = "Apache-2.0"
 __copyright__ = "Copyright 2019, 2020, 2021, 2022, 2023 Yam"

pnlp/piop.py

@@ -212,6 +212,7 @@ def write_json(fpath: str, data, **kwargs) -> None:
     fout.close()
 
 
+
 def write_file(fpath: str, data, **kwargs) -> None:
     with open(fpath, "w", **kwargs) as fout:
         for line in data:

pnlp/pmag.py

@@ -1,5 +1,6 @@
 from collections import Counter
 from addict import Dict
+from pathlib import Path
 
 
 class MagicDict(Dict):
@@ -46,3 +47,20 @@ def reverse(dic):
                            else (x, d2[x])
                            for x in d1.keys()])
         return reversdict
+
+
+def get_unique_fn(file_path: str, level=0):
+    fp = Path(file_path)
+    fn = fp.name
+
+    file_path = str(file_path).strip("/")
+    tmp = file_path.split("/")[:-1]
+    length = len(tmp)
+    if level > length:
+        level = length
+
+    if length == 0:
+        return fn
+    
+    path = "_".join(tmp[-level:])
+    return "_".join([path, fn])

pnlp/utils.py

@@ -145,3 +145,12 @@ def wrapper(lst: List[Any], *args, **kwargs):
             raise ValueError(err_info)
 
     return wrapper
+
+
+def run_in_new_thread(
+    func: Callable, *args, **kwargs
+):
+    if kwargs:
+        func = partial(func, **kwargs)
+    t = Thread(target=func, name="BackgroundRun", args=args)
+    t.start()

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="pnlp",
-    version="0.4.12",
+    version="0.4.13",
     author="Yam",
     author_email="[email protected]",
     description="A pre/post-processing tool for NLP.",

tests/test_pmag.py

@@ -1,4 +1,5 @@
-from pnlp.pmag import MagicDict
+import pytest
+from pnlp.pmag import MagicDict, get_unique_fn
 
 
 def test_magic():
@@ -16,3 +17,23 @@ def test_magic_set_get():
 def test_magic_reverse():
     dx = {1: "a", 2: "a", 3: "a", 4: "b"}
     assert MagicDict.reverse(dx) == {"a": [1, 2, 3], "b": 4}
+
+
+
+@pytest.mark.parametrize("inp,oup,level", [
+    ("a.md", "a.md", 0),
+    ("a.md", "a.md", 1),
+    ("a.md", "a.md", 10),
+    ("a/b.md", "a_b.md", 0),
+    ("a/b.md", "a_b.md", 1),
+    ("a/b.md", "a_b.md", 10),
+    ("a/b/c.md", "a_b_c.md", 0),
+    ("a/b/c.md", "b_c.md", 1),
+    ("a/b/c.md", "a_b_c.md", 10),
+    ("/a/b/c.md", "a_b_c.md", 0),
+    ("/a/b/c.md", "b_c.md", 1),
+    ("/a/b/c.md", "a_b_c.md", 10),
+])
+def test_get_unique_fn(inp, level, oup):
+    res = get_unique_fn(inp, level)
+    assert res == oup

tests/test_utils.py

@@ -1,10 +1,14 @@
 import re
+import os
+from functools import partial
 import math
 import itertools
 import pytest
 import multiprocessing as mp
 
 from pnlp.utils import pstr, concurring, generate_batches_by_num
+from pnlp.utils import run_in_new_thread
+from pnlp.piop import read_file, write_file
 
 
 def test_pstr1():
@@ -119,3 +123,33 @@ def get_primes(lst):
             return res
     except Exception as err:
         assert "0" in str(err)
+
+
+
+def test_run_in_thread():
+    file = "run_in_new_thread.txt"
+    
+    def func(file, a, b, c):
+        write_file(file, list(map(str, [a, b, c])))
+
+    run_in_new_thread(func, file, 1, 2, 3)
+    import time
+    time.sleep(1)
+
+    assert os.path.exists(file)
+    os.remove(file)
+
+
+def test_run_in_thread_kwargs():
+    kwargs = {
+        "b": 2,
+        "c": 3,
+    }
+
+    def func(a, b, c):
+        return a + b + c
+
+    func = partial(func, **kwargs)
+
+    assert 6 == func(1)
+