-
Notifications
You must be signed in to change notification settings - Fork 0
/
convert.py
55 lines (42 loc) · 1.53 KB
/
convert.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
import argparse
from functools import partial
import glob
import logging
from multiprocessing import Pool
import os
import subprocess
from tqdm import tqdm
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--source', required=True,
help='Source directory with publication PDFs')
parser.add_argument('--destination', required=True,
help='Destination directory for publication TXTs')
return parser.parse_args()
def convert_pdf(src: str, dst: str):
try:
completed = subprocess.run(['pdf2txt.py', '-o', dst, src], timeout=300)
if completed.returncode:
logging.warning(f"{src} conversion failed.")
except subprocess.TimeoutExpired:
logging.warning(f"{src} conversion timeout.")
def prepare_destination(src: str, dst_root: str):
head, fname = os.path.split(src)
_, category = os.path.split(head)
pub_name, _ = os.path.splitext(fname)
dst_dir = os.path.join(dst_root, category)
os.makedirs(dst_dir, exist_ok=True)
dst = os.path.join(dst_dir, f"{pub_name}.txt")
return dst
def convert(src: str, dst_root: str):
dst = prepare_destination(src, dst_root)
convert_pdf(src, dst)
def main():
args = parse_args()
pattern = os.path.join(args.source, '**', '*.pdf')
queue = glob.glob(pattern, recursive=True)
_convert = partial(convert, dst_root=args.destination)
with Pool(20) as pool:
pool.map(_convert, tqdm(queue, 'conversion'))
if __name__ == '__main__':
main()