From ae3d433479e02da9b55e571d9440670e3e45da2a Mon Sep 17 00:00:00 2001 From: crocodilestick <105450872+crocodilestick@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:10:58 +0000 Subject: [PATCH] # Version 2.1.0 - Made it possible to shift click to select multiple books in the the Book List page. Courtesy of @jmarmstrong1207 - Added greater support for special characters in Book Titles and Author Names - Improved error handling for files that are unable to be successfully processed - Fix for bug where the Web UI could become unavailable due to not receiving a response for a API query to the project's GitHub page. Courtesy of @Buco7854 - Made it so CWA only checks for available updates once per day - This and future builds now available for ARM based machines thanks to help from @driftywinds and @carlossgv - Enabled Uploads to be on by default - Added default path to included calibre & kepubify binaries - Reworked the ingest process to solve issues for a number of users - Deprecated new-book-detector and renamed scripts to make their function clearer --- .gitignore | 1 + README.md | 6 +- empty_library/app.db | Bin 118784 -> 118784 bytes root/app/calibre-web/cps/cwa_functions.py | 26 +-- .../dependencies.d/auto-library | 0 .../run | 11 +- .../type | 0 .../up | 0 .../s6-rc.d/cwa-init-remove-locks/run | 2 +- .../dependencies.d/init-custom-files | 0 .../{set-cwa-perms => cwa-set-perms}/run | 0 .../etc/s6-overlay/s6-rc.d/cwa-set-perms/type | 1 + root/etc/s6-overlay/s6-rc.d/cwa-set-perms/up | 1 + .../s6-overlay/s6-rc.d/new-book-detector/run | 37 ---- .../s6-overlay/s6-rc.d/new-book-detector/type | 1 - .../s6-overlay/s6-rc.d/new-book-detector/up | 1 - .../etc/s6-overlay/s6-rc.d/set-cwa-perms/type | 1 - root/etc/s6-overlay/s6-rc.d/set-cwa-perms/up | 1 - .../contents.d/cwa-ingest-service} | 0 ...ooks-to-process-detector => cwa-set-perms} | 0 .../s6-rc.d/user/contents.d/new-book-detector | 0 .../s6-rc.d/user/contents.d/set-cwa-perms | 0 scripts/auto-library.py | 2 +- scripts/check-cwa-install.sh | 27 +-- scripts/convert-library.py | 12 +- scripts/ingest-processor.py | 170 ++++++++++++++++++ scripts/new-book-processor.py | 147 --------------- scripts/setup-cwa.sh | 16 +- 28 files changed, 229 insertions(+), 234 deletions(-) rename root/etc/s6-overlay/s6-rc.d/{books-to-process-detector => cwa-ingest-service}/dependencies.d/auto-library (100%) rename root/etc/s6-overlay/s6-rc.d/{books-to-process-detector => cwa-ingest-service}/run (50%) rename root/etc/s6-overlay/s6-rc.d/{books-to-process-detector => cwa-ingest-service}/type (100%) rename root/etc/s6-overlay/s6-rc.d/{books-to-process-detector => cwa-ingest-service}/up (100%) rename root/etc/s6-overlay/s6-rc.d/{set-cwa-perms => cwa-set-perms}/dependencies.d/init-custom-files (100%) rename root/etc/s6-overlay/s6-rc.d/{set-cwa-perms => cwa-set-perms}/run (100%) create mode 100644 root/etc/s6-overlay/s6-rc.d/cwa-set-perms/type create mode 100644 root/etc/s6-overlay/s6-rc.d/cwa-set-perms/up delete mode 100644 root/etc/s6-overlay/s6-rc.d/new-book-detector/run delete mode 100644 root/etc/s6-overlay/s6-rc.d/new-book-detector/type delete mode 100644 root/etc/s6-overlay/s6-rc.d/new-book-detector/up delete mode 100644 root/etc/s6-overlay/s6-rc.d/set-cwa-perms/type delete mode 100644 root/etc/s6-overlay/s6-rc.d/set-cwa-perms/up rename root/etc/s6-overlay/s6-rc.d/{new-book-detector/dependencies.d/auto-library => user/contents.d/cwa-ingest-service} (100%) rename root/etc/s6-overlay/s6-rc.d/user/contents.d/{books-to-process-detector => cwa-set-perms} (100%) delete mode 100644 root/etc/s6-overlay/s6-rc.d/user/contents.d/new-book-detector delete mode 100644 root/etc/s6-overlay/s6-rc.d/user/contents.d/set-cwa-perms create mode 100644 scripts/ingest-processor.py delete mode 100644 scripts/new-book-processor.py diff --git a/.gitignore b/.gitignore index 7516513..686ac14 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +.vscode # PyInstaller # Usually these files are written by a python script from a template diff --git a/README.md b/README.md index 754d119..1eed68b 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ After discovering that using the DOCKER_MODS universal-calibre environment varia - Calibre-Web Automated has always been designed with `.epub` libraries in mind due to many factors, chief among which being the fact they are **Compatible with the Widest Range of Devices**, **Ubiquitous** as well as being **Easy to Manage and Work with** - Previously this meant that anyone with `non-epub` ebooks in their existing Calibre Libraries was unable to take advantage of all of `Calibre-Web Automated`'s features reliably - So new to Version 1.2.0 is the ability for those users to quickly and easily convert their existing eBook Libraries, no matter the size, to `.epub Version 3` format using a one-step CLI Command from within the CWA Container - - This utility gives the user the option to either keep a copy of the original of all converted files in `/config/original-library` or to trust the process and have CWA simply convert and replace those files (not recommended) + - This utility gives the user the option to either keep a copy of the original of all converted files in `/config/processed_books` or to trust the process and have CWA simply convert and replace those files (not recommended) - Full usage details can be found [here](#the-convert-library-tool) - **Simple CLI Tools** for manual fixes, conversions, enforcements, history viewing ect. 👨‍💻 @@ -221,7 +221,7 @@ services: - If your Calibre Library contains any ebooks not in the `.epub` format, from within the container run the `convert-library` command. - Calibre-Web Automated's extra features only work with epubs and so **failure to complete this step could result in unforeseen errors and general unreliability** - - Full usage can be found below in the Usage Section however the following command will automatically convert any non-epubs to epubs and store the original files in `/config/original-library`: + - Full usage can be found below in the Usage Section however the following command will automatically convert any non-epubs to epubs and store the original files in `/config/processed_books`: ``` convert-library --keep @@ -268,7 +268,7 @@ Made for the purpose of converting ebooks in a calibre library not in epub forma options: -h, --help show this help message and exit --replace, -r Replaces the old library with the new one - --keep, -k Creates a new epub library with the old one but stores the old files in /config/original-library + --keep, -k Creates a new epub library with the old one but stores the old files in /config/processed_books -setup Indicates to the function whether or not it's being ran from the setup script or manually (DO NOT USE MANUALLY) ``` diff --git a/empty_library/app.db b/empty_library/app.db index 65dd8e7e727cc42cefe2506a1b518ac1972983e7..9074993277a528fb471650c4c77a97a96790bb09 100644 GIT binary patch delta 113 zcmV-%0FM8Fpa+1U2ap>9A(0$I1t9<~c8P&xwPXRJuLPw44W+Sxn+*blSeMVe0TvB6 z0RadIfjI~P4G1v_m-4*_i!7R#^tUaIeOcttpI)*0Y>t;61ysaSyK!lMMq1v=}g^Gw3pM z=rS<4vv3G9FmNz1F)=XkDl;;2a5P$SF!FL3a!fz9mr 100: - flash(_(f"Library Refresh: Ingest process complete. {return_val - 100} new books ingested."), category="cwa_refresh") - elif return_val == 2: - flash(_("Library Refersh: The book ingest service is already running, please wait until it has finished before trying again."), category="cwa_refresh") -# elif return_val == 0: -# flash(_("Manually starting ingest proces"), category="info") - elif return_val == 0: + result = subprocess.run('python3', '/app/calibre-web-automated/scripts/ingest-processor.py', '/cwa-book-ingest') + return_code = result.returncode + + if return_code > 100: + flash(_(f"Library Refresh: Ingest process complete. {return_code - 100} new books ingested."), category="cwa_refresh") + elif return_code == 2: + flash(_("Library Refresh: The book ingest service is already running, please wait until it has finished before trying again."), category="cwa_refresh") +# elif return_code == 0: +# flash(_("Manually starting ingest process"), category="info") + elif return_code == 0: flash(_("Library Refresh: Ingest process complete. No new books ingested."), category="cwa_refresh") else: - flash(_("Library Refresh: An unexpected error occured, check the logs."), category="cwa_refresh") + flash(_("Library Refresh: An unexpected error occurred, check the logs."), category="cwa_refresh") return redirect("/", code=302) @@ -69,7 +69,7 @@ def cwa_library_refresh(): @admin_required def cwa_library_convert(): flash(_("Library Convert: Running, please wait..."), category="refresh-cwa") - os.system('python3 /app/calibre-web-automated/scripts/convert-library.py -k') + subprocess.Popen('python3', '/app/calibre-web-automated/scripts/convert-library.py', '-k') return redirect(url_for('admin.view_logfile')) # Coming Soon diff --git a/root/etc/s6-overlay/s6-rc.d/books-to-process-detector/dependencies.d/auto-library b/root/etc/s6-overlay/s6-rc.d/cwa-ingest-service/dependencies.d/auto-library similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/books-to-process-detector/dependencies.d/auto-library rename to root/etc/s6-overlay/s6-rc.d/cwa-ingest-service/dependencies.d/auto-library diff --git a/root/etc/s6-overlay/s6-rc.d/books-to-process-detector/run b/root/etc/s6-overlay/s6-rc.d/cwa-ingest-service/run similarity index 50% rename from root/etc/s6-overlay/s6-rc.d/books-to-process-detector/run rename to root/etc/s6-overlay/s6-rc.d/cwa-ingest-service/run index 0edba79..65ead58 100644 --- a/root/etc/s6-overlay/s6-rc.d/books-to-process-detector/run +++ b/root/etc/s6-overlay/s6-rc.d/cwa-ingest-service/run @@ -4,17 +4,16 @@ # This script is used to automatically import downloaded ebooks into a Calibre database. # Reference: https://manual.calibre-ebook.com/generated/en/calibredb.html#add -echo "========== STARTING BOOKS-TO-PROCESS DETECTOR ==========" +echo "========== STARTING CWA-INGEST SERVICE ==========" -# Folder to monitor, replace "/books/to_process" with the folder you want to monitor e.g. your download folder for books WATCH_FOLDER=$(grep -o '"ingest_folder": "[^"]*' /app/calibre-web-automated/dirs.json | grep -o '[^"]*$') -echo "[books-to-process]: Watching folder: $WATCH_FOLDER" +echo "[cwa-ingest-service]: Watching folder: $WATCH_FOLDER" # Monitor the folder for new files s6-setuidgid abc inotifywait -m -r --format="%e %w%f" -e close_write -e moved_to "$WATCH_FOLDER" | while read -r events filepath ; do - echo "[books-to-process]: New files detected - $filepath" - python3 /app/calibre-web-automated/scripts/new-book-processor.py "$filepath" - echo "[books-to-process]: New files successfully moved/converted, the Ingest Folder has been emptied and is ready to go again." + echo "[cwa-ingest-service]: New files detected - $filepath" + python3 /app/calibre-web-automated/scripts/ingest-processor.py "$filepath" + echo "[cwa-ingest-service]: New files successfully moved/converted, the Ingest Folder has been emptied and is ready to go again." done diff --git a/root/etc/s6-overlay/s6-rc.d/books-to-process-detector/type b/root/etc/s6-overlay/s6-rc.d/cwa-ingest-service/type similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/books-to-process-detector/type rename to root/etc/s6-overlay/s6-rc.d/cwa-ingest-service/type diff --git a/root/etc/s6-overlay/s6-rc.d/books-to-process-detector/up b/root/etc/s6-overlay/s6-rc.d/cwa-ingest-service/up similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/books-to-process-detector/up rename to root/etc/s6-overlay/s6-rc.d/cwa-ingest-service/up diff --git a/root/etc/s6-overlay/s6-rc.d/cwa-init-remove-locks/run b/root/etc/s6-overlay/s6-rc.d/cwa-init-remove-locks/run index adeebfe..dd5a4dc 100644 --- a/root/etc/s6-overlay/s6-rc.d/cwa-init-remove-locks/run +++ b/root/etc/s6-overlay/s6-rc.d/cwa-init-remove-locks/run @@ -1,6 +1,6 @@ #!/bin/bash -declare -a lockFiles=("new-book-processor.lock") +declare -a lockFiles=("ingest-processor.lock") echo "[cwa-init-remove-locks] Checking for leftover lock files from prvious instance..." diff --git a/root/etc/s6-overlay/s6-rc.d/set-cwa-perms/dependencies.d/init-custom-files b/root/etc/s6-overlay/s6-rc.d/cwa-set-perms/dependencies.d/init-custom-files similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/set-cwa-perms/dependencies.d/init-custom-files rename to root/etc/s6-overlay/s6-rc.d/cwa-set-perms/dependencies.d/init-custom-files diff --git a/root/etc/s6-overlay/s6-rc.d/set-cwa-perms/run b/root/etc/s6-overlay/s6-rc.d/cwa-set-perms/run similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/set-cwa-perms/run rename to root/etc/s6-overlay/s6-rc.d/cwa-set-perms/run diff --git a/root/etc/s6-overlay/s6-rc.d/cwa-set-perms/type b/root/etc/s6-overlay/s6-rc.d/cwa-set-perms/type new file mode 100644 index 0000000..3d92b15 --- /dev/null +++ b/root/etc/s6-overlay/s6-rc.d/cwa-set-perms/type @@ -0,0 +1 @@ +oneshot \ No newline at end of file diff --git a/root/etc/s6-overlay/s6-rc.d/cwa-set-perms/up b/root/etc/s6-overlay/s6-rc.d/cwa-set-perms/up new file mode 100644 index 0000000..7771e7d --- /dev/null +++ b/root/etc/s6-overlay/s6-rc.d/cwa-set-perms/up @@ -0,0 +1 @@ +/etc/s6-overlay/s6-rc.d/cwa-set-perms/run diff --git a/root/etc/s6-overlay/s6-rc.d/new-book-detector/run b/root/etc/s6-overlay/s6-rc.d/new-book-detector/run deleted file mode 100644 index b7892d5..0000000 --- a/root/etc/s6-overlay/s6-rc.d/new-book-detector/run +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# https://github.com/janeczku/calibre-web/wiki/Automatically-import-new-books-(Linux) - -# This script is used to automatically import downloaded eBook's into a Calibre database. -# Reference: https://manual.calibre-ebook.com/generated/en/calibredb.html#add -echo "============== STARTING NEW BOOK DETECTOR ==============" - -# Folder to monitor -WATCH_FOLDER=$(grep -o '"import_folder": "[^"]*' /app/calibre-web-automated/dirs.json | grep -o '[^"]*$') -echo "[new-book-detector]: Watching folder: $WATCH_FOLDER" - -# Calibre library path -CALIBRE_LIBRARY=$(grep -o '"calibre_library_dir": "[^"]*' /app/calibre-web-automated/dirs.json | grep -o '[^"]*$') - -# Function to add new eBook to Calibre database -add_to_calibre() { - # Path to calibredb executable - CALIBREDB="/usr/bin/calibredb" - - # Run calibredb command to add the new eBook to the database - $CALIBREDB add -r "$1" --library-path="$CALIBRE_LIBRARY" - echo "[new-book-detector] Added $1 to Calibre database" -} - -# Monitor the folder for new files -s6-setuidgid abc inotifywait -m -r --format="%e %w%f" -e close_write -e moved_to "$WATCH_FOLDER" | -while read -r events filepath; do - echo "[new-book-detector]: New file detected: $filepath" - add_to_calibre "$filepath" - echo "[new-book-detector]: Removing $filepath from import folder..." - chown -R abc:users "$WATCH_FOLDER" - #find "$WATCH_FOLDER/" -mindepth 1 -type f,d -delete - rm "$filepath" - chown -R abc:abc "$CALIBRE_LIBRARY" - echo "[new-book-detector]: $filepath successfully moved/converted, the Ingest Folder has been emptied and is ready" -done diff --git a/root/etc/s6-overlay/s6-rc.d/new-book-detector/type b/root/etc/s6-overlay/s6-rc.d/new-book-detector/type deleted file mode 100644 index 1780f9f..0000000 --- a/root/etc/s6-overlay/s6-rc.d/new-book-detector/type +++ /dev/null @@ -1 +0,0 @@ -longrun \ No newline at end of file diff --git a/root/etc/s6-overlay/s6-rc.d/new-book-detector/up b/root/etc/s6-overlay/s6-rc.d/new-book-detector/up deleted file mode 100644 index bb13112..0000000 --- a/root/etc/s6-overlay/s6-rc.d/new-book-detector/up +++ /dev/null @@ -1 +0,0 @@ -run bash.sh \ No newline at end of file diff --git a/root/etc/s6-overlay/s6-rc.d/set-cwa-perms/type b/root/etc/s6-overlay/s6-rc.d/set-cwa-perms/type deleted file mode 100644 index bdd22a1..0000000 --- a/root/etc/s6-overlay/s6-rc.d/set-cwa-perms/type +++ /dev/null @@ -1 +0,0 @@ -oneshot diff --git a/root/etc/s6-overlay/s6-rc.d/set-cwa-perms/up b/root/etc/s6-overlay/s6-rc.d/set-cwa-perms/up deleted file mode 100644 index 75587e5..0000000 --- a/root/etc/s6-overlay/s6-rc.d/set-cwa-perms/up +++ /dev/null @@ -1 +0,0 @@ -/etc/s6-overlay/s6-rc.d/set-cwa-perms/run diff --git a/root/etc/s6-overlay/s6-rc.d/new-book-detector/dependencies.d/auto-library b/root/etc/s6-overlay/s6-rc.d/user/contents.d/cwa-ingest-service similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/new-book-detector/dependencies.d/auto-library rename to root/etc/s6-overlay/s6-rc.d/user/contents.d/cwa-ingest-service diff --git a/root/etc/s6-overlay/s6-rc.d/user/contents.d/books-to-process-detector b/root/etc/s6-overlay/s6-rc.d/user/contents.d/cwa-set-perms similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/user/contents.d/books-to-process-detector rename to root/etc/s6-overlay/s6-rc.d/user/contents.d/cwa-set-perms diff --git a/root/etc/s6-overlay/s6-rc.d/user/contents.d/new-book-detector b/root/etc/s6-overlay/s6-rc.d/user/contents.d/new-book-detector deleted file mode 100644 index e69de29..0000000 diff --git a/root/etc/s6-overlay/s6-rc.d/user/contents.d/set-cwa-perms b/root/etc/s6-overlay/s6-rc.d/user/contents.d/set-cwa-perms deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/auto-library.py b/scripts/auto-library.py index 54153b7..0e8fa1e 100644 --- a/scripts/auto-library.py +++ b/scripts/auto-library.py @@ -44,7 +44,7 @@ def metadb_path(self, path): self._metadb_path = path self.lib_path = os.path.dirname(path) - # Checks config_dir for an exisiting app.db, if one doesn't already exist it copies an empty one from /app/calibre-web-automated/empty_library/app.db and sets the permissiosn + # Checks config_dir for an existing app.db, if one doesn't already exist it copies an empty one from /app/calibre-web-automated/empty_library/app.db and sets the permissions def check_for_app_db(self): files_in_config = [os.path.join(dirpath,f) for (dirpath, dirnames, filenames) in os.walk(self.config_dir) for f in filenames] db_files = [f for f in files_in_config if "app.db" in f] diff --git a/scripts/check-cwa-install.sh b/scripts/check-cwa-install.sh index 879610d..eeb0239 100644 --- a/scripts/check-cwa-install.sh +++ b/scripts/check-cwa-install.sh @@ -8,20 +8,20 @@ NC='\033[0m' # No Color echo "====== Calibre-Web Automated -- Status of Monitoring Services ======" echo "" -if s6-rc -a list | grep -q 'new-book-detector'; then - echo -e "- new-book-detector ${GREEN}is running${NC}" - nb=true -else - echo -e "- new-book-detector ${RED}is not running${NC}" - nb=false -fi +# if s6-rc -a list | grep -q 'new-book-detector'; then +# echo -e "- new-book-detector ${GREEN}is running${NC}" +# nb=true +# else +# echo -e "- new-book-detector ${RED}is not running${NC}" +# nb=false +# fi -if s6-rc -a list | grep -q 'books-to-process-detector'; then - echo -e "- books-to-process-detector ${GREEN}is running${NC}" - bp=true +if s6-rc -a list | grep -q 'cwa-ingest-service'; then + echo -e "- cwa-ingest-service ${GREEN}is running${NC}" + is=true else - echo -e "- books-to-process-detector ${RED}is not running${NC}" - bp=false + echo -e "- cwa-ingest-service ${RED}is not running${NC}" + is=false fi if s6-rc -a list | grep -q 'metadata-change-detector'; then @@ -32,9 +32,10 @@ else mc=false fi + echo "" -if $cs && $bs && $mc; then +if $is && $mc; then echo -e "Calibre-Web-Automated was ${GREEN}successfully installed ${NC}and ${GREEN}is running properly!${NC}" else echo -e "Calibre-Web-Automated was ${RED}not installed successfully${NC}, please check the logs for more information." diff --git a/scripts/convert-library.py b/scripts/convert-library.py index 05e8fbb..776eab0 100644 --- a/scripts/convert-library.py +++ b/scripts/convert-library.py @@ -13,7 +13,7 @@ def __init__(self, args) -> None: self.args = args self.supported_book_formats = ['azw', 'azw3', 'azw4', 'cbz', 'cbr', 'cb7', 'cbc', 'chm', 'djvu', 'docx', 'epub', 'fb2', 'fbz', 'html', 'htmlz', 'lit', 'lrf', 'mobi', 'odt', 'pdf', 'prc', 'pdb', 'pml', 'rb', 'rtf', 'snb', 'tcr', 'txt', 'txtz'] - self.hierarchy_of_succsess = ['lit', 'mobi', 'azw', 'azw3', 'fb2', 'fbz', 'azw4', 'prc', 'odt', 'lrf', 'pdb', 'cbz', 'pml', 'rb', 'cbr', 'cb7', 'cbc', 'chm', 'djvu', 'snb', 'tcr', 'pdf', 'docx', 'rtf', 'html', 'htmlz', 'txtz', 'txt'] + self.hierarchy_of_success = ['lit', 'mobi', 'azw', 'azw3', 'fb2', 'fbz', 'azw4', 'prc', 'odt', 'lrf', 'pdb', 'cbz', 'pml', 'rb', 'cbr', 'cb7', 'cbc', 'chm', 'djvu', 'snb', 'tcr', 'pdf', 'docx', 'rtf', 'html', 'htmlz', 'txtz', 'txt'] self.dirs = self.get_dirs() # Dirs are assigned by user during setup self.import_folder = f"{self.dirs['import_folder']}/" @@ -28,7 +28,7 @@ def get_library_books(self): epub_files = [f for f in library_files if f.endswith('.epub')] dupe_list = [] to_convert = [] - for format in self.hierarchy_of_succsess: + for format in self.hierarchy_of_success: format_files = [f for f in library_files if f.endswith(f'.{format}')] if len(format_files) > 0: for file in format_files: @@ -54,16 +54,16 @@ def convert_library(self): filename, file_extension = os.path.splitext(file) filename = filename.split('/')[-1] book_id = (re.search(r'\(\d*\)', file).group(0))[1:-1] - os.system(f"cp '{file}' '/config/original-library/{filename}{file_extension}'") + os.system(f"cp '{file}' '/config/processed_books/{filename}{file_extension}'") os.system(f"calibredb remove {book_id} --permanent --with-library '{self.library}'") - os.system(f"ebook-convert '/config/original-library/{filename}{file_extension}' '{self.import_folder}{filename}.epub'") # >>/config/calibre-web.log 2>&1 + os.system(f"ebook-convert '/config/processed_books/{filename}{file_extension}' '{self.import_folder}{filename}.epub'") # >>/config/calibre-web.log 2>&1 # if self.args.setup == True: # os.system(f"calibredb add --with-library '{self.library}' '{self.import_folder}{filename}.epub' >>/config/calibre-web.log 2>&1") os.system(f"chown -R abc:abc '{self.library}'") logging.info(f"[convert-library]: Conversion of {os.path.basename(file)} complete!") self.current_book += 1 if not self.args.keep: - os.remove(f"/config/original-library/{filename}{file_extension}") + os.remove(f"/config/processed_books/{filename}{file_extension}") def empty_import_folder(self): os.system(f"chown -R abc:abc '{self.import_folder}'") @@ -78,7 +78,7 @@ def main(): ) parser.add_argument('--replace', '-r', action='store_true', required=False, dest='replace', help='Replaces the old library with the new one', default=False) - parser.add_argument('--keep', '-k', action='store_true', required=False, dest='keep', help='Creates a new epub library with the old one but stores the old files in /config/original-library', default=False) + parser.add_argument('--keep', '-k', action='store_true', required=False, dest='keep', help='Creates a new epub library with the old one but stores the old files in /config/processed_books', default=False) # parser.add_argument('-setup', action='store_true', required=False, dest='setup', help="Indicates to the function whether or not it's being ran from the setup script or manually (DO NOT USE MANUALLY)", default=False) args = parser.parse_args() diff --git a/scripts/ingest-processor.py b/scripts/ingest-processor.py new file mode 100644 index 0000000..20e1efc --- /dev/null +++ b/scripts/ingest-processor.py @@ -0,0 +1,170 @@ +import atexit +import json +import os +import subprocess +import sys +import tempfile +import time +import shutil + +from pathlib import Path + +# Global variable counting the number of books processed +books_processed = 0 + +# Used to generate a count of the number of books processed during each run +# 1 book is 101 as there are functions that use the scripts exit code to tell the number of processed books +# In that case, the code being over 100 indicates at least one book was processed and the actual number is the value - 100 +def increment_books_processed(): + global books_processed + if books_processed == 0: + books_processed = 101 + else: + books_processed += 1 + +# Creates a lock file unless one already exists meaning an instance of the script is +# already running, then the script is closed, the user is notified and the program +# exits with code 2 +try: + lock = open(tempfile.gettempdir() + '/ingest-processor.lock', 'x') + lock.close() +except FileExistsError: + print("CANCELLING... ingest-processor initiated but is already running") + sys.exit(2) + +# Defining function to delete the lock on script exit +def removeLock(): + os.remove(tempfile.gettempdir() + '/ingest-processor.lock') + +# Will automatically run when the script exits +atexit.register(removeLock) + +class NewBookProcessor: + def __init__(self, filepath: str): + self.supported_book_formats = ['azw', 'azw3', 'azw4', 'cbz', 'cbr', 'cb7', 'cbc', 'chm', 'djvu', 'docx', 'epub', 'fb2', 'fbz', 'html', 'htmlz', 'lit', 'lrf', 'mobi', 'odt', 'pdf', 'prc', 'pdb', 'pml', 'rb', 'rtf', 'snb', 'tcr', 'txtz'] + self.hierarchy_of_success = ['lit', 'mobi', 'azw', 'epub', 'azw3', 'fb2', 'fbz', 'azw4', 'prc', 'odt', 'lrf', 'pdb', 'cbz', 'pml', 'rb', 'cbr', 'cb7', 'cbc', 'chm', 'djvu', 'snb', 'tcr', 'pdf', 'docx', 'rtf', 'html', 'htmlz', 'txtz'] + self.ingest_folder, self.library_dir = self.get_dirs("/app/calibre-web-automated/dirs.json") + self.tmp_conversion_dir = "/config/.cwa_conversion_tmp/" + + self.filepath = filepath # path of the book we're targeting + self.filename = os.path.basename(filepath) + self.is_epub: bool = bool(self.filepath.endswith('.epub')) + + def get_dirs(self, dirs_json_path: str) -> tuple[str, str, str]: + dirs = {} + with open(dirs_json_path, 'r') as f: + dirs: dict[str, str] = json.load(f) + + ingest_folder = f"{dirs['ingest_folder']}/" # Dir where new files are looked for to process and subsequently deleted + library_dir = f"{dirs['calibre_library_dir']}/" + + return ingest_folder, library_dir + + + def convert_book(self, import_format: str) -> tuple[bool, str]: + """Uses the following terminal command to convert the books provided using the calibre converter tool:\n\n--- ebook-convert myfile.input_format myfile.output_format\n\nAnd then saves the resulting epubs to the calibre-web import folder.""" + + print(f"[ingest-processor]: START_CON: Converting {self.filename}...\n") + original_filepath = Path(self.filepath) + target_filepath = f"{self.tmp_conversion_dir}{original_filepath.stem}.epub" + + try: + t_convert_book_start = time.time() + subprocess.run('ebook-convert', f'"{self.filepath}"', f'"{target_filepath}"', check=True) + t_convert_book_end = time.time() + time_book_conversion = t_convert_book_end - t_convert_book_start + print(f"\n[ingest-processor]: END_CON: Conversion of {self.filename} complete in {time_book_conversion:.2f} seconds.\n") + shutil.copyfile(self.filepath, f"/config/processed_books/converted/{original_filepath.stem}") + return True, target_filepath + except subprocess.CalledProcessError as e: + print(f"[ingest-processor]: CON_ERROR: {self.filename} could not be converted to epub due to the following error:\nEXIT/ERROR CODE: {e.returncode}\n{e.stderr}") + shutil.copyfile(self.filepath, f"/config/processed_books/failed/{original_filepath.stem}") + return False, "" + + + def can_convert_check(self): + """When no epubs are detected in the download, this function will go through the list of new files + and check for the format the are in that has the highest chance of successful conversion according to the input format hierarchy list + provided by calibre""" + can_convert = False + import_format = '' + for format in self.hierarchy_of_success: + can_be_converted = bool(self.filepath.endswith(f'.{format}')) + if can_be_converted: + can_convert = True + import_format = format + break + + return can_convert, import_format + + + def delete_current_file(self) -> None: + """Deletes file just processed from ingest folder""" + os.remove(self.filepath) # Removes processed file + subprocess.run(["find", f"{self.ingest_folder}", "-type", "d", "-empty", "-delete"]) # Removes any now empty folders + + + def add_book_to_library(self, book_path) -> None: + print("[ingest-processor]: Importing new epub to CWA...") + import_path = Path(book_path) + import_filename = os.path.basename(book_path) + try: + subprocess.run("calibredb", "add", f"{book_path}", f"--library-path='{self.library_dir}'", check=True) + print(f"[ingest-processor] Added {import_path.stem} to Calibre database") + shutil.copyfile(book_path, f"/config/processed_books/imported/{import_filename}") + except subprocess.CalledProcessError as e: + print(f"[ingest-processor] {import_path.stem} was not able to be added to the Calibre Library due to the following error:\nCALIBREDB EXIT/ERROR CODE: {e.returncode}\n{e.stderr}") + shutil.copyfile(book_path, f"/config/processed_books/failed/{import_filename}") + + + def empty_tmp_con_dir(self): + try: + files = os.listdir(self.tmp_conversion_dir) + for file in files: + file_path = os.path.join(self.tmp_conversion_dir, file) + if os.path.isfile(file_path): + os.remove(file_path) + except OSError: + print(f"Error occurred while emptying {self.tmp_conversion_dir}.") + + +def main(filepath=sys.argv[1]): + # Check if filepath is a directory + # If it is, main will be ran on every file in the given directory + # Inotifywait won't detect files inside folders if the folder was moved rather than copied + if os.path.isdir(filepath): + print(os.listdir(filepath)) + for filename in os.listdir(filepath): + f = os.path.join(filepath, filename) + main(f) + return + + nbp = NewBookProcessor(filepath) + + if not nbp.is_epub: # Books require conversion + print(f"\n[ingest-processor]: Starting conversion process for {self.filename}...") + can_convert, import_format = nbp.can_convert_check() + print(f"[ingest-processor]: Converting file from {import_format} to epub format...\n") + + if can_convert: + result, epub_filepath = nbp.convert_book(import_format) + if result: + nbp.add_book_to_library(epub_filepath) + increment_books_processed() + nbp.empty_tmp_con_dir() + else: + print(f"[ingest-processor]: Cannot convert {nbp.filepath}. {import_format} is currently unsupported.") + + else: # Books need imported + print(f"\n[ingest-processor]: No conversion needed for {self.filename}, importing now...") + npb.add_book_to_library(filepath) + # nbp.move_epub() + increment_books_processed() + + nbp.delete_current_file() + del nbp # New in Version 2.0.0, should drastically reduce memory usage with large ingests + +if __name__ == "__main__": + main() + print(f"[ingest-processor] All {books_processed} books found in ingest folder processed! Exiting now...") + sys.exit(books_processed) \ No newline at end of file diff --git a/scripts/new-book-processor.py b/scripts/new-book-processor.py deleted file mode 100644 index 44858ad..0000000 --- a/scripts/new-book-processor.py +++ /dev/null @@ -1,147 +0,0 @@ -import atexit -import json -import os -import subprocess -import sys -import tempfile -import time - -# Global variable counting the number of books processed -books_processed = 0 - -def increment_books_processed(): - global books_processed - if books_processed == 0: - books_processed = 101 - else: - books_processed += 1 - -# Creates a lock file unless one already exists meaning an instance of the script is -# already running, then the script is closed, the user is notified and the program -# exits with code 2 -try: - lock = open(tempfile.gettempdir() + '/new-book-processor.lock', 'x') - lock.close() -except FileExistsError: - print("CANCELING... new-book-processor initiated but is already running") - sys.exit(2) - -# Defineing function to delete the lock on script exit -def removeLock(): - os.remove(tempfile.gettempdir() + '/new-book-processor.lock') - -# Will automatically run when the script exits -atexit.register(removeLock) - -class NewBookProcessor: - def __init__(self, filepath: str): - self.supported_book_formats = ['azw', 'azw3', 'azw4', 'cbz', 'cbr', 'cb7', 'cbc', 'chm', 'djvu', 'docx', 'epub', 'fb2', 'fbz', 'html', 'htmlz', 'lit', 'lrf', 'mobi', 'odt', 'pdf', 'prc', 'pdb', 'pml', 'rb', 'rtf', 'snb', 'tcr', 'txtz'] - self.hierarchy_of_success = ['lit', 'mobi', 'azw', 'epub', 'azw3', 'fb2', 'fbz', 'azw4', 'prc', 'odt', 'lrf', 'pdb', 'cbz', 'pml', 'rb', 'cbr', 'cb7', 'cbc', 'chm', 'djvu', 'snb', 'tcr', 'pdf', 'docx', 'rtf', 'html', 'htmlz', 'txtz'] - self.import_folder, self.ingest_folder = self.get_dirs("/app/calibre-web-automated/dirs.json") - - self.filepath = filepath # path of the book we're targeting - self.is_epub: bool = bool(self.filepath.endswith('.epub')) - - def get_dirs(self, dirs_json_path: str) -> tuple[str, str]: - dirs = {} - with open(dirs_json_path, 'r') as f: - dirs: dict[str, str] = json.load(f) - # Both folders are preassigned in dirs.json but can be changed with the 'cwa-change-dirs' command from within the container - import_folder = f"{dirs['import_folder']}/" - ingest_folder = f"{dirs['ingest_folder']}/" # Dir where new files are looked for to process and subsequently deleted - - return import_folder, ingest_folder - - - def convert_book(self, import_format: str) -> float: - """Uses the following terminal command to convert the books provided using the calibre converter tool:\n\n--- ebook-convert myfile.input_format myfile.output_format\n\nAnd then saves the resulting epubs to the calibre-web import folder.""" - t_convert_total_start = time.time() - t_convert_book_start = time.time() - filename = self.filepath.split('/')[-1] - print(f"[new-book-processor]: START_CON: Converting {filename}...\n") - os.system(f'ebook-convert "{self.filepath}" "{self.import_folder}{(filename.split(f".{import_format}"))[0]}.epub"') - t_convert_book_end = time.time() - time_book_conversion = t_convert_book_end - t_convert_book_start - print(f"\n[new-book-processor]: END_CON: Conversion of {filename} complete in {time_book_conversion:.2f} seconds.\n") - - t_convert_total_end = time.time() - time_total_conversion = t_convert_total_end - t_convert_total_start - - return time_total_conversion - - - def can_convert_check(self): - """When no epubs are detected in the download, this function will go through the list of new files - and check for the format the are in that has the highest chance of successful conversion according to the input format hierarchy list - provided by calibre""" - can_convert = False - import_format = '' - for format in self.hierarchy_of_success: - can_be_converted = bool(self.filepath.endswith(f'.{format}')) - if can_be_converted: - can_convert = True - import_format = format - break - - return can_convert, import_format - - def move_epub(self) -> None: - """Moves the epubs from the download folder to the calibre-web import folder""" - print(f"[new-book-processor]: Moving {self.filepath}...") - filename = self.filepath.split('/')[-1] - os.system(f'cp "{self.filepath}" "{self.import_folder}{filename}"') - - def empty_to_process_folder(self) -> None: - """Empties the ingest folder""" - os.remove(self.filepath) - subprocess.run(["find", f"{self.ingest_folder}", "-type", "d", "-empty", "-delete"]) - - def delete_file(self) -> None: - """Empties the ingest folder""" - os.remove(self.filepath) - subprocess.run(["find", f"{self.ingest_folder}", "-type", "d", "-empty", "-delete"]) - - -def main(filepath=sys.argv[1]): - # Check if filepath is a directory - # If it is, main will be ran on every file in the given directory - # Inotifywait won't detect files inside folders if the folder was moved rather than copied - if os.path.isdir(filepath): - print(os.listdir(filepath)) - for filename in os.listdir(filepath): - f = os.path.join(filepath, filename) - main(f) - return - - # t_start = time.time() - nbp = NewBookProcessor(filepath) - - if not nbp.is_epub: # Books require conversion - print("\n[new-book-processor]: No epub files found in the current directory. Starting conversion process...") - can_convert, import_format = nbp.can_convert_check() - print(f"[new-book-processor]: Converting file from to epub format...\n") - - if can_convert: - time_total_conversion = nbp.convert_book(import_format) - print(f"\n[new-book-processor]: Conversion to .epub format completed succsessfully in {time_total_conversion:.2f} seconds.") - print("[new-book-processor]: Importing new epub to CWA...") - increment_books_processed() - else: - print(f"Cannot convert {nbp.filepath}") - - else: # Books only need copying to the import folder - print(f"\n[new-book-processor]: Found epub file in ingest folder.") - print("[new-book-processor]: Moving epub file to the CWA import folder...\n") - nbp.move_epub() - increment_books_processed() - - # t_end = time.time() - # running_time = t_end - t_start - - # print(f"[new-book-processor]: Processing of new files completed in {running_time:.2f} seconds.\n\n") - nbp.delete_file() - del nbp # New in Version 2.0.0, should drastically reduce memory usage with large ingests - -if __name__ == "__main__": - main() - sys.exit(books_processed) \ No newline at end of file diff --git a/scripts/setup-cwa.sh b/scripts/setup-cwa.sh index 005ac2b..48943e9 100644 --- a/scripts/setup-cwa.sh +++ b/scripts/setup-cwa.sh @@ -12,15 +12,25 @@ make_dirs () { chown abc:abc /cwa-book-ingest mkdir /calibre-library chown abc:abc /calibre-library + mkdir /config/.cwa_conversion_tmp + chown abc:abc /config/.cwa_conversion_tmp + mkdir /config/processed_books + chown abc:abc /config/processed_books + mkdir /config/processed_books/imported + chown abc:abc /config/processed_books/imported + mkdir /config/processed_books/failed + chown abc:abc /config/processed_books/failed + mkdir /config/processed_books/converted + chown abc:abc /config/processed_books/converted } # Change ownership & permissions as required change_script_permissions () { chmod +x /app/calibre-web-automated/scripts/check-cwa-install.sh - chmod +x /etc/s6-overlay/s6-rc.d/books-to-process-detector/run - chmod +x /etc/s6-overlay/s6-rc.d/new-book-detector/run + chmod +x /etc/s6-overlay/s6-rc.d/cwa-ingest-service/run + # chmod +x /etc/s6-overlay/s6-rc.d/new-book-detector/run chmod +x /etc/s6-overlay/s6-rc.d/metadata-change-detector/run - chmod +x /etc/s6-overlay/s6-rc.d/set-cwa-perms/run + chmod +x /etc/s6-overlay/s6-rc.d/cwa-set-perms/run chmod +x /etc/s6-overlay/s6-rc.d/auto-library/run chmod 775 /app/calibre-web/cps/editbooks.py chmod 775 /app/calibre-web/cps/admin.py