abstract_cluster.py

# -*- coding: utf-8 -*-
"""Untitled11.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/10YsYsZnP4jzpkptIzqGPP6Sf2UiE2SZ7
"""

!pip install bibtexparser pandas sentence-transformers umap-learn matplotlib

import bibtexparser
import pandas as pd

# Load .bib file
with open("savedrecs.bib", encoding="utf-8") as bibtex_file:
    bib_database = bibtexparser.load(bibtex_file)

# Convert entries to DataFrame
df_bib = pd.DataFrame(bib_database.entries)

# Show basic structure
df_bib[['title', 'abstract']].dropna().head()

from sentence_transformers import SentenceTransformer
import umap
import matplotlib.pyplot as plt

# Load transformer model
model = SentenceTransformer('allenai-specter')  # SciBERT model trained for science

# Clean and extract abstracts
abstracts = df_bib['abstract'].dropna().tolist()

# Load transformer model
model = SentenceTransformer('allenai-specter')  # SciBERT model trained for science

# Encode abstracts
embeddings = model.encode(abstracts, show_progress_bar=True)

# Dimensionality reduction for plotting
reducer = umap.UMAP(n_neighbors=10, min_dist=0.3, metric='cosine')
embedding_2d = reducer.fit_transform(embeddings)

# Visualize
plt.figure(figsize=(10, 7))
plt.scatter(embedding_2d[:, 0], embedding_2d[:, 1], alpha=0.6)
plt.title("Semantic Clustering of Abstracts")
plt.xlabel("UMAP-1")
plt.ylabel("UMAP-2")
plt.grid(True)
plt.show()