sync changes:

- M idx/file_to_keywords.json - M idx/keyword_to_files.json - M scripts/keywords.py
elimelt · Jan 13, 2025 · 4df94fb · 4df94fb
1 parent 19d406c
commit 4df94fb
Show file tree

Hide file tree

Showing 113 changed files with 149 additions and 329 deletions.
diff --git a/idx/file_to_keywords.json b/idx/file_to_keywords.json
diff --git a/idx/keyword_to_files.json b/idx/keyword_to_files.json
diff --git a/scripts/keywords.py b/scripts/keywords.py
@@ -2,12 +2,13 @@
 import string
 import nltk
 
+from nltk.stem import WordNetLemmatizer
 from typing import List
 from pathlib import Path
 from keybert import KeyBERT
 
 nltk.download("stopwords")
-
+nltk.download('wordnet')
 
 class DataReader:
     def __init__(self, root_dir: str = "."):
@@ -83,6 +84,21 @@ def get_kw_path_map(files: List[Path], model: KeyBERT) -> dict:
 
     return keyword_map
 
+def deduplicate(file_to_keywords: dict) -> dict:
+    wnl = WordNetLemmatizer()
+    deduped = {}
+    dedup_count = 0
+    for f, kws in file_to_keywords.items():
+        deduped[f] = []
+        for kw, acc in kws:
+            stem = wnl.lemmatize(kw)
+            if stem != kw:
+                print('stem', stem, 'kw', kw)
+                dedup_count += 1
+            if stem not in deduped[f]:
+                deduped[f].append((stem, acc))
+    print('dedup count', dedup_count)
+    return deduped
 
 def aggregate(file_to_keywords: dict) -> dict:
     keyword_to_files = {}
@@ -110,7 +126,7 @@ def write_idx_to_json_file(idx: dict, output_path: Path):
     files = reader.read_files()
     model = KeyBERT()
 
-    file_to_kw = get_kw_path_map(files, model)
+    file_to_kw = deduplicate(get_kw_path_map(files, model))
     kw_to_file = aggregate(file_to_kw)
     write_idx_to_json_file(file_to_kw, Path("idx/file_to_keywords.json"))
     write_idx_to_json_file(kw_to_file, Path("idx/keyword_to_files.json"))
diff --git a/scripts/simple.md b/scripts/simple.md
diff --git a/site/categories/algorithm analysis.html b/site/categories/algorithm analysis.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: Algorithm Analysis</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/algorithms.html b/site/categories/algorithms.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: algorithms</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/computer science.html b/site/categories/computer science.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: Computer Science</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/database design.html b/site/categories/database design.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: Database Design</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/database systems.html b/site/categories/database systems.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: Database Systems</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/distributed systems.html b/site/categories/distributed systems.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: Distributed Systems</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/graph theory.html b/site/categories/graph theory.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: Graph Theory</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/index.html b/site/categories/index.html
@@ -182,7 +182,7 @@
         </div>
         <h1>Categories</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/mathematics.html b/site/categories/mathematics.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: Mathematics</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/operations research.html b/site/categories/operations research.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: Operations Research</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/research.html b/site/categories/research.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: research</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/software engineering.html b/site/categories/software engineering.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: Software Engineering</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/system-design.html b/site/categories/system-design.html
@@ -183,7 +183,7 @@
         </div>
         <h1>Category: system-design</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">

diff --git a/site/idx/file_to_keywords.json b/site/idx/file_to_keywords.json
diff --git a/site/idx/keyword_to_files.json b/site/idx/keyword_to_files.json
diff --git a/site/index.html b/site/index.html
@@ -182,14 +182,14 @@
         </div>
         <h1></h1>
         <div class="meta">
-            <span>Last modified: 2025-01-12</span>
+            <span>Last modified: 2025-01-13</span>
 
         </div>
         <div class="content">
 
             <div class="landing-stats">
                 <div class="stat-item">
-                    <span class="stat-value">152</span>
+                    <span class="stat-value">151</span>
                     <span class="stat-label">Notes</span>
                 </div>
                 <div class="stat-item">

diff --git a/site/scripts/keywords.py b/site/scripts/keywords.py
@@ -2,12 +2,13 @@
 import string
 import nltk
 
+from nltk.stem import WordNetLemmatizer
 from typing import List
 from pathlib import Path
 from keybert import KeyBERT
 
 nltk.download("stopwords")
-
+nltk.download('wordnet')
 
 class DataReader:
     def __init__(self, root_dir: str = "."):
@@ -83,6 +84,21 @@ def get_kw_path_map(files: List[Path], model: KeyBERT) -> dict:
 
     return keyword_map
 
+def deduplicate(file_to_keywords: dict) -> dict:
+    wnl = WordNetLemmatizer()
+    deduped = {}
+    dedup_count = 0
+    for f, kws in file_to_keywords.items():
+        deduped[f] = []
+        for kw, acc in kws:
+            stem = wnl.lemmatize(kw)
+            if stem != kw:
+                print('stem', stem, 'kw', kw)
+                dedup_count += 1
+            if stem not in deduped[f]:
+                deduped[f].append((stem, acc))
+    print('dedup count', dedup_count)
+    return deduped
 
 def aggregate(file_to_keywords: dict) -> dict:
     keyword_to_files = {}
@@ -110,7 +126,7 @@ def write_idx_to_json_file(idx: dict, output_path: Path):
     files = reader.read_files()
     model = KeyBERT()
 
-    file_to_kw = get_kw_path_map(files, model)
+    file_to_kw = deduplicate(get_kw_path_map(files, model))
     kw_to_file = aggregate(file_to_kw)
     write_idx_to_json_file(file_to_kw, Path("idx/file_to_keywords.json"))
     write_idx_to_json_file(kw_to_file, Path("idx/keyword_to_files.json"))