config.json

pyproject.toml downloadNewArticles.py generateLists.py utils.py uv.lock
distbit0 · Jan 22, 2025 · cb139e6 · cb139e6
1 parent 7c2df07
commit cb139e6
Show file tree

Hide file tree

Showing 7 changed files with 899 additions and 30 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
diff --git a/config.json b/config.json
@@ -59,6 +59,22 @@
         ".stfolder"
     ],
     "listToFolderMappings": {
+        "PDFs": {
+            "formats": [
+                "pdf"
+            ]
+        },
+        "EPUBs": {
+            "formats": [
+                "epub",
+                "mobi"
+            ]
+        },
+        "YT": {
+            "folders": [
+                "YTvideo"
+            ]
+        }
         "interpersonal": {
             "folders": [
                 "Philosophy/conversation and relationships"

diff --git a/pyproject.toml b/pyproject.toml
@@ -5,10 +5,27 @@ description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.13"
 dependencies = [
+    "audioop-lts>=0.2.1",
+    "bs4>=0.0.2",
+    "cid>=0.1.3",
     "eldar>=0.0.8",
+    "html2text>=2024.2.26",
+    "markdown>=3.7",
     "matplotlib>=3.10.0",
+    "multihash>=0.1.1",
+    "openai>=1.59.9",
+    "pdftitle>=0.17",
+    "py-cid>=0.3.0",
+    "pydub>=0.25.1",
     "pypdf2>=3.0.1",
+    "pyperclip>=1.9.0",
     "pysnooper>=1.2.1",
+    "python-dotenv>=1.0.1",
+    "secretstorage>=3.3.3",
+    "selenium>=4.28.0",
+    "soundcloud-lib>=0.6.1",
+    "telethon>=1.38.1",
     "unidecode>=1.3.8",
     "urlexpander>=0.0.37",
+    "yt-dlp>=2025.1.15",
 ]
diff --git a/src/downloadNewArticles.py b/src/downloadNewArticles.py
@@ -36,7 +36,6 @@ def save_text_as_html(url):
 def downloadNewArticles(urlsToAdd):
     saveDirectory = getConfig()["pdfSourceFolders"][0]
     print(urlsToAdd)
-    downloaded_urls = []
     for url in urlsToAdd:
         if url.endswith(".pdf"):
             continue
@@ -46,10 +45,9 @@ def downloadNewArticles(urlsToAdd):
         except Exception as e:
             print("Error downloading article: ", url, e)
         else:
-            downloaded_urls.append(url)
+            addUrlToUrlFile([url], getAbsPath("../storage/alreadyAddedArticles.txt"))
 
     # Add downloaded URLs to alreadyAddedArticles.txt
-    addUrlToUrlFile(downloaded_urls, getAbsPath("../storage/alreadyAddedArticles.txt"))
 
 
 def save_webpage_as_mhtml(url, timeout=10, min_load_time=5):

diff --git a/src/generateLists.py b/src/generateLists.py
@@ -8,11 +8,12 @@ def updateLists():
     listFolderMappings = getConfig()["listToFolderMappings"]
 
     for listName, listInfo in listFolderMappings.items():
-        listFolders, readState, query, disabled = (
+        listFolders, readState, query, disabled, formats = (
             listInfo.get("folders", []),
             listInfo.get("readState", "unread"),
             listInfo.get("query", "*"),
             listInfo.get("disabled", False),
+            listInfo.get("formats", getConfig()["docFormatsToMove"]),
         )
         if disabled:
             utils.deleteListIfExists(listName)
@@ -23,7 +24,7 @@ def updateLists():
             query,
             subjects=listFolders,
             readState=readState,
-            formats=getConfig()["docFormatsToMove"],
+            formats=formats,
         )
 
         articlePathsForList = [

diff --git a/src/utils.py b/src/utils.py
@@ -20,7 +20,7 @@
 def checkArticleSubject(articlePath, subjects):
     if not subjects:
         return True
-    articlePath = "/".join(articlePath.split("/")[:-1])
+    # articlePath = "/".join(articlePath.split("/")[:-1]) commented out because sometimes I want to filter by the filename e.g. to find yt videos
     for subject in subjects:
         if subject.lower() in articlePath.lower():
             return True