sync changes:

- M "site/categories/algorithm analysis.html" - M site/categories/algorithms.html - M "site/categories/computer science.html"
elimelt · Jan 12, 2025 · bcedc2b · bcedc2b
1 parent e60f6ed
commit bcedc2b
Show file tree

Hide file tree

Showing 95 changed files with 223 additions and 95 deletions.
diff --git a/site/categories/algorithm analysis.html b/site/categories/algorithm analysis.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Category: Algorithm Analysis</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/algorithms.html b/site/categories/algorithms.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Category: algorithms</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/computer science.html b/site/categories/computer science.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Category: Computer Science</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/database design.html b/site/categories/database design.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Category: Database Design</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/database systems.html b/site/categories/database systems.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Category: Database Systems</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/distributed systems.html b/site/categories/distributed systems.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Category: Distributed Systems</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/graph theory.html b/site/categories/graph theory.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Category: Graph Theory</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/index.html b/site/categories/index.html
@@ -178,7 +178,7 @@
         </div>
         <h1>Categories</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/mathematics.html b/site/categories/mathematics.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Category: Mathematics</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/operations research.html b/site/categories/operations research.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Category: Operations Research</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/categories/software engineering.html b/site/categories/software engineering.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Category: Software Engineering</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/index.html b/site/index.html
@@ -178,7 +178,7 @@
         </div>
         <h1>My Notes</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">
@@ -203,7 +203,7 @@ <h2>Recent Notes</h2>
 <ul class='recent-posts'>
                 <li>
                     <a href="/systems-research/hints-for-computer-system-design.html">Hints For Computer System Design</a>
-                    <span class="date">2025-01-08</span>
+                    <span class="date">2025-01-09</span>
 
                 </li>
                 <li>

diff --git a/site/natural-language-processing/tf-idf.py b/site/natural-language-processing/tf-idf.py
@@ -0,0 +1,128 @@
+import numpy as np
+import os
+from typing import Dict, List, Tuple
+
+
+def read_markdown_files(root_dir: str, max_depth: int = 8) -> Dict[str, str]:
+
+    corpus = {}
+
+    def process_dir(path: str, depth: int = 0):
+        if depth > max_depth:
+            return
+
+        try:
+            for entry in os.listdir(path):
+                full_path = os.path.join(path, entry)
+                if os.path.isdir(full_path):
+                    process_dir(full_path, depth + 1)
+                elif entry.endswith(".md"):
+                    try:
+                        with open(full_path, "r") as f:
+                            corpus[full_path] = f.read()
+                    except Exception:
+                        continue
+        except Exception:
+            return
+
+    process_dir(root_dir)
+    return corpus
+
+
+def create_index(corpus: Dict[str, str]) -> Tuple[Dict[str, Dict[str, int]], List[str]]:
+
+    inv_idx = {}
+    for doc_name, content in corpus.items():
+        words = content.split()
+        for word in words:
+            if word not in inv_idx:
+                inv_idx[word] = {}
+            inv_idx[word][doc_name] = inv_idx[word].get(doc_name, 0) + 1
+
+    return inv_idx, list(inv_idx.keys())
+
+
+def calculate_tfidf(
+    corpus: Dict[str, str], inv_idx: Dict[str, Dict[str, int]], word_list: List[str]
+) -> Dict[str, np.ndarray]:
+
+    N = len(corpus)
+    tfidf = {}
+
+    for doc_name, content in corpus.items():
+        doc_words = content.split()
+        doc_len = len(doc_words)
+        tfidf[doc_name] = np.zeros(len(word_list))
+
+        for i, word in enumerate(word_list):
+            if word in inv_idx and doc_name in inv_idx[word]:
+                tf = inv_idx[word][doc_name] / doc_len
+                idf = np.log(N / len(inv_idx[word]))
+                tfidf[doc_name][i] = tf * idf
+
+    return tfidf
+
+
+def search(
+    query: str,
+    tfidf: Dict[str, np.ndarray],
+    word_list: List[str],
+    inv_idx: Dict[str, Dict[str, int]],
+    num_results: int = 5,
+) -> List[Tuple[str, float]]:
+
+    N = len(tfidf)
+    query_vec = np.zeros(len(word_list))
+
+    for i, word in enumerate(word_list):
+        if word in query.split() and word in inv_idx:
+            query_vec[i] = np.log(N / len(inv_idx[word]))
+
+    similarities = [
+        (doc_name, float(np.dot(doc_vec, query_vec)))
+        for doc_name, doc_vec in tfidf.items()
+    ]
+
+    return sorted(similarities, key=lambda x: x[1], reverse=True)[:num_results]
+
+
+def main():
+
+    try:
+        root_dir = (
+            input("Enter root directory to search (default '.'): ").strip() or "."
+        )
+        print(f"Reading markdown files from {root_dir}...")
+
+        corpus = read_markdown_files(root_dir)
+        if not corpus:
+            print("No markdown files found!")
+            return
+
+        print(f"Found {len(corpus)} markdown files")
+        print("Building search index...")
+
+        inv_idx, word_list = create_index(corpus)
+        tfidf = calculate_tfidf(corpus, inv_idx, word_list)
+
+        print("Search system ready!")
+
+        while True:
+            query = input('\nEnter search query (or "exit" to quit): ').strip()
+            if query.lower() == "exit":
+                break
+
+            results = search(query, tfidf, word_list, inv_idx)
+
+            print("\nSearch results:")
+            for doc_name, similarity in results:
+                print(f"{doc_name}: {similarity:.4f}")
+
+    except KeyboardInterrupt:
+        print("\nExiting...")
+    except Exception as e:
+        print(f"\nError: {str(e)}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/site/systems-research/hints-for-computer-system-design.html b/site/systems-research/hints-for-computer-system-design.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Hints For Computer System Design</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-09</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/acyclic graphs.html b/site/tags/acyclic graphs.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: acyclic graphs</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/algorithm-analysis.html b/site/tags/algorithm-analysis.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: algorithm-analysis</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/algorithm.html b/site/tags/algorithm.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: algorithm</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/algorithms.html b/site/tags/algorithms.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: algorithms</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/approximation.html b/site/tags/approximation.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: approximation</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/asymptotic notation.html b/site/tags/asymptotic notation.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: asymptotic notation</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/batch processing.html b/site/tags/batch processing.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: batch processing</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/bipartite graphs.html b/site/tags/bipartite graphs.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: bipartite graphs</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/bipartite matching.html b/site/tags/bipartite matching.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: bipartite matching</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/breadth-first search.html b/site/tags/breadth-first search.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: breadth-first search</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/column-oriented storage.html b/site/tags/column-oriented storage.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: column-oriented storage</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/compatibility.html b/site/tags/compatibility.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: compatibility</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/complexity analysis.html b/site/tags/complexity analysis.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: complexity analysis</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/complexity-analysis.html b/site/tags/complexity-analysis.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: complexity-analysis</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/connected components.html b/site/tags/connected components.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: connected components</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">

diff --git a/site/tags/connected graphs.html b/site/tags/connected graphs.html
@@ -179,7 +179,7 @@
         </div>
         <h1>Tag: connected graphs</h1>
         <div class="meta">
-            <span>Last modified: 2025-01-08</span>
+            <span>Last modified: 2025-01-11</span>
 
         </div>
         <div class="content">