diff --git a/.gitignore b/.gitignore index 4656b518..bbec2b31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +example_out +example_results +hidden_keys.py example_output/ VOCAB_CONSOLIDATOR_SubstitutionOperator.p VOCAB_CONSOLIDATOR_changes.csv diff --git a/CITATION.cff b/CITATION.cff index 3228c06c..652ad81a 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,4 +1,4 @@ -version: 0.0.41 +version: 0.0.42 message: "If you use this software, please cite it as below." authors: - family-names: Eren @@ -20,7 +20,7 @@ authors: - family-names: Alexandrov given-names: Boian title: "Tensor Extraction of Latent Features (T-ELF)" -version: 0.0.41 +version: 0.0.42 url: https://github.com/lanl/T-ELF doi: 10.5281/zenodo.10257897 date-released: 2023-12-04 diff --git a/README.md b/README.md index d3bf8de6..5d8ce9e9 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,7 @@ python post_install.py # use the following, for example, for GPU system: ` sections. + include_general : bool + If filtering by category, whether to include pre-category terms. + substitutions : bool + If True, builds substitution maps. + all_categories : bool + If True, overrides `category` and `include_general`. + drop_conflicts : bool + If True, resolve substitution conflicts and prune dropped entries. + If False, keep all substitutions as-is (even if conflicting). """ - def __init__(self, markdown_file, lower=False, category=None, - include_general=True, substitutions=False, all_categories=False): + + def __init__( + self, + markdown_file, + lower: bool = False, + category: Optional[str] = None, + include_general: bool = True, + substitutions: bool = False, + all_categories: bool = False, + drop_conflicts: bool = True, + ): self.markdown_file = markdown_file self.lower = lower self.category = category self.include_general = include_general self.substitutions = substitutions self.all_categories = all_categories + self.drop_conflicts = drop_conflicts - self.substitution_forward = {} - self.substitution_reverse = {} + self.substitution_forward: Dict[str, str] = {} + self.substitution_reverse: Dict[str, str] = {} - # parse the markdown into self.terms - self.terms = self._parse_markdown() + # parse markdown → raw terms list + self.terms: List[Any] = self._parse_markdown() - # optionally build lookup table + # optionally build lookup tables if self.substitutions: self._build_substitutions_lookup() + if self.drop_conflicts: + self._postprocess_conflicts() - - def _parse_markdown(self): - terms = [] - current_term = None - positives = [] - negatives = [] - active_block = False - current_section = None + # ──────────────────────────────────────────────────────────────── # + # markdown parsing # + # ──────────────────────────────────────────────────────────────── # + def _parse_markdown(self) -> List[Any]: + terms: List[Any] = [] + current_term, positives, negatives = None, [], [] + active_block, current_section = False, None try: - with open(self.markdown_file, 'r', encoding='utf-8') as f: + with open(self.markdown_file, "r", encoding="utf-8") as f: lines = f.readlines() except FileNotFoundError: warnings.warn(f"File '{self.markdown_file}' not found. Returning empty list.") @@ -52,120 +79,145 @@ def _parse_markdown(self): for raw in lines: line = raw.strip() - # Section header if line.startswith("# Category:"): current_section = line.split(":", 1)[1].strip() continue - # Decide whether to include this section - if self.all_categories: - include_section = True - elif self.category is None: - # no filtering → include everything - include_section = True - else: - if current_section is None and self.include_general: - include_section = True - else: - include_section = (current_section == self.category) + include_section = self.all_categories or self.category is None + if self.category and not self.all_categories: + include_section = (current_section == self.category) or ( + current_section is None and self.include_general + ) - # Term header if line.startswith("##"): - # finish previous block if current_term is not None and active_block: if positives or negatives: - terms.append({ - current_term: { - "positives": positives, - "negatives": negatives - } - }) + terms.append({current_term: {"positives": positives, "negatives": negatives}}) else: terms.append(current_term) - # reset for new block - positives = [] - negatives = [] + positives, negatives = [], [] header = line.lstrip("#").strip() if self.lower: header = header.lower() - current_term = header - active_block = include_section + current_term = header + active_block = include_section - # collect positives / negatives - elif active_block and line.lower().startswith("must have:"): + elif active_block and line.lower().startswith("positives:"): items = [i.strip() for i in line.split(":", 1)[1].split(",") if i.strip()] positives.extend(items) - elif active_block and line.lower().startswith("exclude with:"): + + elif active_block and line.lower().startswith("negatives:"): items = [i.strip() for i in line.split(":", 1)[1].split(",") if i.strip()] negatives.extend(items) - # final block if current_term is not None and active_block: if positives or negatives: - terms.append({ - current_term: { - "positives": positives, - "negatives": negatives - } - }) + terms.append({current_term: {"positives": positives, "negatives": negatives}}) else: terms.append(current_term) return terms - def _build_substitutions_lookup(self): - """ - Build a dict mapping each term to its underscored form and vice versa. - """ + # ──────────────────────────────────────────────────────────────── # + # substitutions lookup # + # ──────────────────────────────────────────────────────────────── # + def _build_substitutions_lookup(self) -> None: + """Create forward & reverse maps (no filtering yet).""" for entry in self.terms: if isinstance(entry, str): term = entry underscored = term.replace(" ", "_") self.substitution_forward[term] = underscored self.substitution_reverse[underscored] = term - elif isinstance(entry, dict): + else: # dict for term in entry.keys(): underscored = term.replace(" ", "_") self.substitution_forward[term] = underscored self.substitution_reverse[underscored] = term + def _postprocess_conflicts(self) -> None: + """Resolve substitution conflicts and prune dropped terms.""" + clean_forward, dropped = resolve_substitution_conflicts( + self.substitution_forward, warn=True + ) + self.substitution_forward = clean_forward + + # rebuild reverse map + rev: Dict[str, List[str]] = {} + for src, tgt in clean_forward.items(): + rev.setdefault(tgt, []).append(src) + self.substitution_reverse = rev + + if not dropped: + return + + # prune self.terms to match cleaned substitutions + pruned_terms: List[Any] = [] + for entry in self.terms: + if isinstance(entry, str): + if entry not in dropped: + pruned_terms.append(entry) + else: + kept = {k: v for k, v in entry.items() if k not in dropped} + if kept: + pruned_terms.append(kept) + self.terms = pruned_terms + + # ──────────────────────────────────────────────────────────────── # + # public access # + # ──────────────────────────────────────────────────────────────── # + def get_terms(self) -> List[Any]: + return self.terms + + def get_substitution_maps(self) -> Tuple[Dict[str, str], Dict[str, str]]: + return self.substitution_forward, self.substitution_reverse + + + # ──────────────────────────────────────────────────────────────── # + # public helpers # + # ──────────────────────────────────────────────────────────────── # def get_terms(self): return self.terms def get_substitution_maps(self): - """ - Return the substitutions lookup dict (empty if substitutions=False). - """ + """Return (forward_map, reverse_map).""" return self.substitution_forward, self.substitution_reverse +# ═══════════════════════════════════════════════════════════════════ # +# utility: convert TXT dump → cheetah markdown # +# ═══════════════════════════════════════════════════════════════════ # def convert_txt_to_cheetah_markdown(txt_path, markdown_path): + """ + Helper to convert a simple TXT list (optionally containing dict literals) + into the markdown format expected by CheetahTermFormatter. + """ import ast - with open(txt_path, 'r', encoding='utf-8') as f: + with open(txt_path, "r", encoding="utf-8") as f: lines = [line.strip() for line in f if line.strip()] - markdown_lines = [] + md_lines: List[str] = [] for line in lines: if line.startswith("{") and line.endswith("}"): try: parsed = ast.literal_eval(line) for key, value in parsed.items(): - positives = [v.lstrip('+') for v in value if v.startswith('+')] - negatives = [v for v in value if not v.startswith('+')] - markdown_lines.append(f"## {key}") + positives = [v.lstrip("+") for v in value if v.startswith("+")] + negatives = [v for v in value if not v.startswith("+")] + md_lines.append(f"## {key}") if positives: - markdown_lines.append(f"positives: {', '.join(positives)}") + md_lines.append(f"positives: {', '.join(positives)}") if negatives: - markdown_lines.append(f"negatives: {', '.join(negatives)}") + md_lines.append(f"negatives: {', '.join(negatives)}") except Exception as e: print(f"Skipping line due to parse error: {line}\nError: {e}") else: - markdown_lines.append(f"## {line.strip()}") + md_lines.append(f"## {line.strip()}") - with open(markdown_path, 'w', encoding='utf-8') as f: - f.write("\n".join(markdown_lines)) + with open(markdown_path, "w", encoding="utf-8") as f: + f.write("\n".join(md_lines)) print(f"Converted markdown saved to: {markdown_path}") diff --git a/TELF/applications/Lynx/__init__.py b/TELF/applications/Lynx/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/TELF/applications/Lynx/backend/__init__.py b/TELF/applications/Lynx/backend/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/TELF/applications/Lynx/frontend/__init__.py b/TELF/applications/Lynx/frontend/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/TELF/applications/Lynx/frontend/main.py b/TELF/applications/Lynx/frontend/main.py new file mode 100644 index 00000000..fa8c069f --- /dev/null +++ b/TELF/applications/Lynx/frontend/main.py @@ -0,0 +1,21 @@ +import streamlit as st +import os +import sys;sys.path.append(os.path.join("pages")) +import sys;sys.path.append(os.path.join("..", "backend")) +import sys;sys.path.append(os.path.join("..")) + +if "project_loaded" not in st.session_state: + st.session_state.project_loaded = False + +load_project_page = st.Page(os.path.join("pages", "load_project.py"), title="Load Project", icon=":material/flag:", default=True) +tree_view_page = st.Page(os.path.join("pages", "tree_view.py"), title="Tree Search", icon=":material/allergy:", default=False) +document_analysis_view_page = st.Page(os.path.join("pages", "doc_view.py"), title="Document Analysis", icon=":material/lan:", default=False) +link_view_page = st.Page(os.path.join("pages", "link_view.py"), title="Link Prediction", icon=":material/linked_services:", default=False) + +pg = st.navigation( + { + f"Lynx":[load_project_page], + "Views":[tree_view_page, document_analysis_view_page, link_view_page], + } +) +pg.run() diff --git a/TELF/applications/Lynx/frontend/pages/__init__.py b/TELF/applications/Lynx/frontend/pages/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/TELF/applications/Lynx/frontend/pages/doc_view.py b/TELF/applications/Lynx/frontend/pages/doc_view.py new file mode 100644 index 00000000..2da74f55 --- /dev/null +++ b/TELF/applications/Lynx/frontend/pages/doc_view.py @@ -0,0 +1,252 @@ +import streamlit as st +from streamlit_tree_select import tree_select +import pandas as pd +import os +import numpy as np +from pages.helpers.utils import (find_files_by_extensions, + prune_tree, filter_documents, + extract_unique_values) +from pages.helpers.displays import (display_html, + display_dataframe, display_wordcloud, + open_explorer_button, display_images_in_tabs, + display_html_selector, display_csv_in_tab, + display_document_years, display_attributes, + display_denovo_words, display_author_chart, + display_interactive_coauthorship_plot, + display_document_affiliations, + display_coauthorship_heatmap) + + +st.set_page_config(layout="wide") +if ("project_loaded" in st.session_state and st.session_state.project_loaded == False) or "data" not in st.session_state: + st.warning("⚠️ Data has not been loaded yet. Please load the data before accessing this page.") + st.stop() + +if st.session_state.view_type != "Document Analysis": + st.warning("⚠️ Document Analysis view mode is not selected.") + st.stop() + +st.header(st.session_state.selected_project) +with st.sidebar: + st.header("Token Search") + + top_n = st.selectbox("Top n:", ["All"] + list(range(1, 101)), index=0) + keywords_search_type = st.selectbox("Word Search Type:", ["Keywords", "Denovo"], index=0) + if keywords_search_type == "Keywords": + token_search_field = "all_words" + else: + token_search_field = "all_words_denovo" + + # Gather all unique words + words = list( + set( + w + for wlist in st.session_state.documents_data[token_search_field].values() + for w in wlist + ) + ) + words.sort() + + selected_words = st.multiselect("Select words to search:", words) + negative_words = st.multiselect("Select negative words:", words) + token_search_type = st.radio("Search type:", ["and", "or"]) + + # --------------------------------------------------- + # Attribute Search + st.header("Attribute Search") + + search_categories = { + "Countries": { + "options": extract_unique_values("all_countries", st.session_state.documents_data), + "key": "country" + }, + "Authors": { + "options": extract_unique_values("all_authors", st.session_state.documents_data), + "key": "author" + }, + "Affiliations": { + "options": extract_unique_values("all_affiliations", st.session_state.documents_data), + "key": "affiliation" + }, + f"Attribute {st.session_state.attribute_column}": { + "options": extract_unique_values("all_attributes", st.session_state.documents_data), + "key": "attributes" + } + } + + # Build UI for attribute selections + attr_selections = {} + for label, info in search_categories.items(): + with st.expander(f"Search by {label}"): + attr_key = info["key"] + attr_selections[f"selected_{attr_key}"] = st.multiselect( + f"Select {label.lower()}:", info["options"] + ) + attr_selections[f"{attr_key}_search_type"] = st.radio( + "Search type:", ["and", "or"], key=f"{attr_key}_radio" + ) + # --------------------------------------------------- + # Other Settings + st.header("Other Settings") + filter_df_setting = st.checkbox("Filter Dataframe", value=True, key="filter_dataframe_settings") + + +# --------------------------------------------------------------------- +# Determine if the user selected anything +any_token_selected = bool(selected_words) +any_negative_token_selected = bool(negative_words) +any_attr_selected = any( + attr_selections.get(f"selected_{cfg['key']}", []) + for cfg in search_categories.values() +) + +# If nothing is selected, reset the data +if not any_token_selected and not any_attr_selected and not negative_words: + st.session_state.data = st.session_state.data_original.copy() + st.session_state.selected_indices = None +else: + # Filter documents once based on both token and attribute selections + selected_labels, selected_indices = filter_documents( + data_map=st.session_state.data_map, + top_n=top_n, + selected_words=selected_words, + token_search_type=token_search_type, + attr_selections=attr_selections, + search_categories=search_categories, + keywords_search_type=keywords_search_type, + use_index_map=filter_df_setting, + negative_words=negative_words, + ) + if selected_labels: + st.session_state.data = [prune_tree(st.session_state.data_original[0], selected_labels, st.session_state.root_name)] + st.session_state.selected_indices = selected_indices + else: + # No matches => clear data or handle how you prefer + st.session_state.data = [{"children":[]}] + st.session_state.selected_indices = None + #st.write(selected_indices) + +selected_nodes = tree_select( + st.session_state.data[0]["children"], + expand_on_click=False, + show_expand_all=True, + check_model="all", + no_cascade=True +) + +tabs = st.tabs(["Words", "Documents", "Peacock", "Attributes", "Authors", "Affiliations"]) +st.session_state.html_figure = None # Reset HTML viewer + +### WORDS TAB +with tabs[0]: + for ii in selected_nodes["checked"]: + directory = st.session_state.data_map[ii]["path"] + with st.expander(st.session_state.data_map[ii]["label"]): + open_explorer_button(directory, key=f"button_tab0_{ii}") + + word_tab_columns = st.columns(3) + # Display unigrams + with word_tab_columns[0]: + display_dataframe(directory, suffix="_bow_unigrams.csv", title="Unigrams", columns=["word", "tf", "df"], ends=True) + + # Display bigrams + with word_tab_columns[1]: + display_dataframe(directory, suffix="_bow_bigrams.csv", title="Bigrams", columns=["word", "tf", "df"], ends=True) + + # Display word cloud + with word_tab_columns[2]: + display_wordcloud(directory) + + if st.session_state.data_map[ii]["all_text"] is not None and len(st.session_state.data_map[ii]["all_text"]) > 0: + display_denovo_words(st.session_state.data_map[ii]["all_text"], key=f"slider_words{[ii]}") + +### DOCUMENTS TAB +with tabs[1]: + for ii in selected_nodes["checked"]: + directory = st.session_state.data_map[ii]["path"] + label = st.session_state.data_map[ii]["label"] + index_filter = None if st.session_state.selected_indices is None else st.session_state.selected_indices[label] + with st.expander(label): + open_explorer_button(directory, key=f"button_tab1_{ii}") + display_dataframe(directory, suffix=st.session_state.document_file_name_suffix, + title="Top Documents", columns=None, ends=False, + index_filter=index_filter) + display_document_years(directory, suffix=st.session_state.document_file_name_suffix, + title="Top Documents Trends", ends=False, column="year", + key=f"tab1{ii}_document_years", index_filter=index_filter) + +### PEACOCK TAB +with tabs[2]: + + for ii in selected_nodes["checked"]: + directory = st.session_state.data_map[ii]["path"] + peacock_dir = os.path.join(directory, "peacock") + files = find_files_by_extensions(peacock_dir, extensions=("html", "png")) + with st.expander(st.session_state.data_map[ii]["label"]): + open_explorer_button(peacock_dir, key=f"button_tab2_{ii}") + all_tabs = [os.path.basename(f) for f in files["png"]] + ["Interactive Plots", "Affiliation", "Author"] + + if all_tabs: + tab_objects = st.tabs(all_tabs) + + # Display images in tabs + display_images_in_tabs(tab_objects, files["png"]) + + # Display interactive plots selector + display_html_selector(tab_objects[-3], files["html"], key=f"select_html_plot{ii}") + + # Display affiliation and author data + display_csv_in_tab(tab_objects[-2], peacock_dir, "_affiliation.csv", "Affiliations Peacock file is not found.", ends=True) + display_csv_in_tab(tab_objects[-1], peacock_dir, "_author.csv", "Authors Peacock file is not found.", ends=True) + +### ATTRIBUTES TAB +with tabs[3]: + for ii in selected_nodes["checked"]: + directory = st.session_state.data_map[ii]["path"] + with st.expander(st.session_state.data_map[ii]["label"]): + open_explorer_button(directory, key=f"button_tab3_{ii}") + if len(st.session_state.data_map[ii]["attributes"]) > 0: + st.header(st.session_state.attribute_column) + display_attributes(st.session_state.data_map[ii]["attributes"], key=f"tab3_attributes_{ii}") + else: + st.warning(f"⚠️ Attributes from column {st.session_state.attribute_column} not found!") + +### AUTHORS TAB +with tabs[4]: + for ii in selected_nodes["checked"]: + directory = st.session_state.data_map[ii]["path"] + with st.expander(st.session_state.data_map[ii]["label"]): + open_explorer_button(directory, key=f"button_tab4_{ii}") + display_author_chart(directory=directory, + suffix=st.session_state.document_file_name_suffix, + title="Author Distribution Chart", + ends=False, columns=["authors", "author_ids"], + key=f"tab4_author_{ii}") + display_interactive_coauthorship_plot(directory=directory, + suffix=st.session_state.document_file_name_suffix, + title="Co-authorship Network", + ends=False, columns=["authors", "author_ids"], + key=f"tab4_coauthor_{ii}") + display_coauthorship_heatmap(directory=directory, + suffix=st.session_state.document_file_name_suffix, + title="Co-authorship Heatmap", + ends=False, columns=["authors", "author_ids"], + key=f"tab4_coauthor_heatmap_{ii}") + + +### AFFILIATIONS TAB +with tabs[5]: + for ii in selected_nodes["checked"]: + directory = st.session_state.data_map[ii]["path"] + with st.expander(st.session_state.data_map[ii]["label"]): + open_explorer_button(directory, key=f"button_tab5_{ii}") + display_document_affiliations(directory=directory, + suffix=st.session_state.document_file_name_suffix, + title="Affiliation Plots", + ends=False, column="affiliations", + key=f"tab5_affiliations_{ii}") + +### DISPLAY HTML COMPONENT FULL WIDTH (NO JAVASCRIPT) +if st.session_state.html_figure: + st.divider() + display_html(path=st.session_state.html_figure, height=800) diff --git a/TELF/applications/Lynx/frontend/pages/helpers/__init__.py b/TELF/applications/Lynx/frontend/pages/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/TELF/applications/Lynx/frontend/pages/helpers/displays.py b/TELF/applications/Lynx/frontend/pages/helpers/displays.py new file mode 100644 index 00000000..5d01e619 --- /dev/null +++ b/TELF/applications/Lynx/frontend/pages/helpers/displays.py @@ -0,0 +1,585 @@ +import streamlit as st +import pandas as pd +from PIL import Image +import streamlit.components.v1 as components +import os +from pages.helpers.utils import (find_by_suffix, open_file_browser, + get_top_words, preprocess_authors, + build_coauthorship_network, extract_all_affiliation_info) +import plotly.express as px +import plotly.graph_objects as go +import numpy as np +from collections import Counter +import networkx as nx + +def st_normal(): + _, col, _ = st.columns([1, 2, 1]) + return col + +def display_html(path, height=800): + st.markdown( + """ + + """, + unsafe_allow_html=True + ) + + with open(path, "r", encoding="utf-8") as f: + html_content = f.read() + + st.markdown('
', unsafe_allow_html=True) + st.markdown(f"**{os.path.basename(path)}**") + components.html(html_content, height=height, scrolling=True) + st.markdown('
', unsafe_allow_html=True) + +def display_denovo_words(data, key): + st.subheader(f"De novo words from '{st.session_state.text_column}' column") + n_gram=st.slider( + "n_gram:", + min_value=1, + max_value=5, + value=2, + key=f"{key}_slider" + ) + st.dataframe(get_top_words(data, + top_n=50, + n_gram=n_gram, + verbose=False, filename=None)[["word", "tf", "df"]], + use_container_width=True) + +def display_coauthorship_heatmap(directory, suffix, ends=False, + columns=["authors", "author_ids"], + title="Co-authorship Heatmap", key=""): + """ + Generates an interactive heatmap of co-authorship counts using Plotly with dynamic filtering. + Filters Y-axis based on user selection and X-axis dynamically adjusts to show only relevant collaborators. + """ + st.subheader(title) + file = find_by_suffix(directory, suffix=suffix, ends=ends) + + if file: + df = pd.read_csv(file) + for col in columns: + if col not in df.columns.tolist(): + st.warning(f"⚠️ {col} not in the DataFrame!") + return + else: + st.warning(f"⚠️ CSV not found!") + return + + coauthorship_counts = {} + + # Process co-authors and count occurrences + for authors, ids in zip(df[columns[0]], df[columns[1]]): + authors_list = preprocess_authors([authors], [ids]) + + for i in range(len(authors_list)): + for j in range(i + 1, len(authors_list)): + pair = tuple(sorted([authors_list[i], authors_list[j]])) + coauthorship_counts[pair] = coauthorship_counts.get(pair, 0) + 1 + + # Convert to DataFrame + coauthorship_df = pd.DataFrame(list(coauthorship_counts.items()), columns=["Pair", "Count"]) + coauthorship_df[["Author 1", "Author 2"]] = coauthorship_df["Pair"].apply(pd.Series) + + # Pivot table for heatmap + coauthorship_matrix = coauthorship_df.pivot(index="Author 1", columns="Author 2", values="Count").fillna(0) + + # Extract unique author names + all_authors = sorted(set(coauthorship_df["Author 1"]).union(set(coauthorship_df["Author 2"]))) + + # Create a searchable dropdown for author selection (Y-axis) + selected_authors = st.multiselect( + "🔍 Select Author(s) to Filter Y-Axis:", + all_authors, + default=[] # No default selection + ) + + # Filter Y-axis based on selection + if selected_authors: + filtered_matrix = coauthorship_matrix.loc[selected_authors, :] + else: + filtered_matrix = coauthorship_matrix # Show all if no selection + + # **Dynamic Filtering for X-axis: Keep only actual collaborators** + if not filtered_matrix.empty: + # Get all actual collaborators of selected authors + relevant_x_authors = filtered_matrix.columns[filtered_matrix.sum(axis=0) > 0].tolist() + filtered_matrix = filtered_matrix.loc[:, relevant_x_authors] # Filter X-axis + else: + relevant_x_authors = coauthorship_matrix.columns.tolist() # Show all if empty + + # Convert filtered matrix to long-form data for Plotly + filtered_long = filtered_matrix.stack().reset_index() + filtered_long.columns = ["Author 1", "Author 2", "Count"] + + # Plot interactive Heatmap using Plotly + fig = px.imshow( + filtered_matrix.values, + labels=dict(x="Co-Author", y="Filtered Author(s)", color="Collaboration Count"), + x=filtered_matrix.columns, + y=filtered_matrix.index, + color_continuous_scale="Blues" + ) + + fig.update_layout( + xaxis_title="Filtered Co-Authors", + yaxis_title="Selected Authors", + width=900, height=700 + ) + + # Display in Streamlit + st.plotly_chart(fig, use_container_width=True, key=f"{key}_coauthor_heatmap") + + + +def display_interactive_coauthorship_plot(directory, suffix, ends=False, + columns=["authors", "author_ids"], + title="Co-authorship Network", key=""): + """ + Streamlit function to display an interactive co-authorship network plot. + """ + st.subheader(title) + file = find_by_suffix(directory, suffix=suffix, ends=ends) + if file: + df = pd.read_csv(file) + for col in columns: + if col not in df.columns.tolist(): + st.warning(f"⚠️ {col} not in the DataFrame!") + return + else: + st.warning(f"⚠️ CSV not found!") + return + # Build graph + G = build_coauthorship_network(df[columns[0]], df[columns[1]]) + + # Dropdown for selecting an author + author_names = list(G.nodes) + selected_author = st.selectbox("Select an author to highlight:", [""] + author_names, key=f"{key}_selectbox1") + + # Dropdown for layout selection + layout_options = { + "Spring Layout": nx.spring_layout, + "Kamada-Kawai Layout": nx.kamada_kawai_layout, + "Circular Layout": nx.circular_layout, + "Fruchterman-Reingold Layout": nx.fruchterman_reingold_layout, + "Random Layout": nx.random_layout, + "Shell Layout": nx.shell_layout + } + selected_layout = st.selectbox("Select network layout:", list(layout_options.keys()), index=0, key=f"{key}_selectbox2") + + # Compute node positions based on selected layout + pos = layout_options[selected_layout](G) + + # Create edge traces + edge_traces = [] + for edge in G.edges(data=True): + x0, y0 = pos[edge[0]] + x1, y1 = pos[edge[1]] + weight = edge[2]["weight"] + + edge_trace = go.Scatter( + x=[x0, x1, None], + y=[y0, y1, None], + line=dict(width=max(0.5, weight / 2), color="gray"), + hoverinfo="text", + mode="lines", + text=f"Co-authored {weight} times" + ) + edge_traces.append(edge_trace) + + # Create node traces + node_x, node_y, node_text, node_color = [], [], [], [] + for node in G.nodes(): + x, y = pos[node] + node_x.append(x) + node_y.append(y) + node_text.append(node) + + if node == selected_author: + node_color.append("red") # Highlight selected author + else: + node_color.append("blue") + + node_trace = go.Scatter( + x=node_x, y=node_y, + mode="markers+text", + text=node_text, + textposition="top center", + marker=dict(size=10, color=node_color, line=dict(width=2, color="black")), + hoverinfo="text" + ) + + # Create figure + fig = go.Figure(data=edge_traces + [node_trace]) + fig.update_layout( + title="Co-authorship Network Graph", + showlegend=False, + hovermode="closest", + margin=dict(b=20, l=5, r=5, t=40), + xaxis=dict(showgrid=False, zeroline=False, showticklabels=False), + yaxis=dict(showgrid=False, zeroline=False, showticklabels=False), + ) + + # Display in Streamlit + st.plotly_chart(fig, use_container_width=True, key=f"{key}_coauthor_network") + + +def display_author_chart(directory, suffix, ends=False, + columns=["authors", "author_ids"], + title="Author Distribution Chart", key=""): + """ + Streamlit function to display an interactive pie chart of author distributions using Plotly. + """ + st.subheader(title) + file = find_by_suffix(directory, suffix=suffix, ends=ends) + if file: + df = pd.read_csv(file) + for col in columns: + if col not in df.columns.tolist(): + st.warning(f"⚠️ {col} not in the DataFrame!") + return + + # Preprocess author names + author_list = preprocess_authors(df[columns[0]], df[columns[1]]) + + # Count occurrences + author_counts = pd.Series(author_list).value_counts().reset_index() + author_counts.columns = ["Author", "Count"] + + # Calculate percentage share + total_publications = author_counts["Count"].sum() + author_counts["Share (%)"] = (author_counts["Count"] / total_publications) * 100 + + # Select number of top authors to display + top_n = st.slider("Select number of top authors to display:", 1, 100, 1, key=f"{key}_slider") + author_counts = author_counts.head(top_n) + + # Create interactive bar chart using Plotly + fig = px.bar( + author_counts, + x="Author", + y="Count", + text="Share (%)", # Show percentage on bars + title="Top Authors by Number of Publications (with Share %)", + labels={"Author": "Author Name", "Count": "Number of Publications"}, + hover_data={"Share (%)": ":.2f"}, + ) + + # Adjust layout for better readability + fig.update_layout( + xaxis_tickangle=-45, + showlegend=False, + yaxis_title="Number of Publications", + ) + + # Display in Streamlit + st.plotly_chart(fig, use_container_width=True, key=f"{key}_author_chart") + else: + st.warning(f"⚠️ CSV not found!") + +def display_document_affiliations(directory, suffix, title, ends, column="affiliations", warning_message="CSV not found!", key=""): + file = find_by_suffix(directory, suffix=suffix, ends=ends) + if file: + st.subheader(title) + df = pd.read_csv(file) + if column not in df.columns.tolist(): + st.warning(f"⚠️ {column} not in the DataFrame!") + return + else: + st.warning(f"⚠️ CSV not found!") + return + organizations, countries = extract_all_affiliation_info(df[column]) + # organizations + org_counts = pd.Series(organizations).value_counts().reset_index() + org_counts.columns = ["Organization", "Count"] + top_n_orgs = st.slider("Select number of top organizations to display:", 1, min(100, len(org_counts)), 1, key=f"{key}_slider1") + org_counts = org_counts.head(top_n_orgs) + fig_orgs = px.bar( + org_counts, + x="Organization", + y="Count", + text="Count", + title="Top Organizations by Count", + labels={"Organization": "Organization Name", "Count": "Number of Publications"}, + ) + fig_orgs.update_layout(xaxis_tickangle=-45, showlegend=False) + + # Display both plots in Streamlit + st.plotly_chart(fig_orgs, use_container_width=True, key=f"{key}_plotly_organization_count") + + # countries + country_counts = pd.Series(countries).value_counts().reset_index() + country_counts.columns = ["Country", "Count"] + top_n_countries = st.slider("Select number of top countries to display:", 1, min(100, len(country_counts)), 1, key=f"{key}_slider2") + country_counts = country_counts.head(top_n_countries) + fig_countries = px.bar( + country_counts, + x="Country", + y="Count", + text="Count", + title="Top Country by Count", + labels={"Country": "Country Name", "Count": "Number of Publications"}, + ) + fig_orgs.update_layout(xaxis_tickangle=-45, showlegend=False) + + # Display both plots in Streamlit + st.plotly_chart(fig_countries, use_container_width=True, key=f"{key}_plotly_country_count") + +def display_document_years(directory, suffix, title, ends, + column="year", warning_message="CSV not found!", key="", + index_filter=None): + file = find_by_suffix(directory, suffix=suffix, ends=ends) + if file: + st.subheader(title) + df = pd.read_csv(file) + if column not in df.columns.tolist(): + st.warning(f"⚠️ {column} not in the DataFrame!") + return + + # ------------------ filter rows by index ------------------ + if index_filter is not None: + # Ensure we’re working with a set for fast lookup + wanted = set(index_filter) + df = df.loc[df.index.intersection(wanted)] + + # Step 1: Convert "Year" column to numeric, handling errors + df[column] = pd.to_numeric(df[column], errors="coerce") # Convert invalid years to NaN + df = df.dropna(subset=[column]) # Remove rows with NaN values + df[column] = df[column].astype(int) # Ensure integer type + + # Step 2: Count occurrences of each year + year_counts = df[column].value_counts().sort_index() + + # Step 3: Ensure no missing years by creating a complete range + year_range = np.arange(year_counts.index.min(), year_counts.index.max() + 1) + df_counts = pd.DataFrame({"Year": year_range, "Value": year_counts.reindex(year_range, fill_value=0)}) + + if df_counts["Year"].min() == df_counts["Year"].max(): + filtered_df = df_counts + else: + year_range_selection = st.slider( + "Select Year Range:", + min_value=df_counts["Year"].min(), + max_value=df_counts["Year"].max(), + value=(df_counts["Year"].min(), df_counts["Year"].max()), + key=f"{key}_slider" + ) + + # 🏷️ Filter DataFrame based on selected years + filtered_df = df_counts[(df_counts["Year"] >= year_range_selection[0]) & (df_counts["Year"] <= year_range_selection[1])] + + # 📊 Create an interactive Plotly line chart + fig = px.line( + filtered_df, + x="Year", + y="Value", + labels={"Year": "Year", "Value": "Count"}, + markers=True + ) + + # 🔍 Display the interactive plot + st.plotly_chart(fig, use_container_width=True, key=f"{key}_document_years") + + else: + st.warning(f"⚠️ {warning_message}") + +def display_link_node_search(link_attributes_df): + + # Multi-select for filtering by 'Node' + selected_nodes = st.multiselect("Select Nodes", options=link_attributes_df["node"].unique(), default=[]) + + # If nothing is selected, show full DataFrame; otherwise, filter by selected nodes + filtered_df = link_attributes_df if not selected_nodes else link_attributes_df[link_attributes_df["node"].isin(selected_nodes)] + + # Multi-select for filtering by other attributes (excluding 'Node') + filterable_columns = [col for col in link_attributes_df.columns if col != "node"] + selected_columns = st.multiselect("Filter by columns", options=filterable_columns, default=[]) + + # Apply column-based filtering if any attribute is selected + for column in selected_columns: + unique_values = filtered_df[column].unique() + selected_values = st.multiselect(f"Select values for {column}", unique_values, default=[]) + if selected_values: # Only filter if values are selected + filtered_df = filtered_df[filtered_df[column].isin(selected_values)] + + # Display the filtered DataFrame + st.dataframe(filtered_df, use_container_width=True) + + +def display_attributes(attribute_list, key=""): + counter = Counter(attribute_list) + df_counts = pd.DataFrame(counter.items(), columns=["Item", "Count"]).sort_values(by="Count", ascending=False) + + # Streamlit App + st.subheader("Pie Chart for Attribute Distribution") + + # Plot Interactive Pie Chart using Plotly + fig = px.pie(df_counts, names="Item", values="Count", + hover_data=["Count"], labels={"Item": "Element", "Count": "Frequency"}, + hole=0.3) # Creates a donut-style pie chart + + # Show the plot in Streamlit + st.plotly_chart(fig, use_container_width=True, key=f"{key}_attributes_pie") + + # Show the data table for reference + st.subheader("Count Data Table") + st.dataframe(df_counts, use_container_width=True) + +def display_dataframe( + directory, + suffix, + title, + columns=["word", "tf", "df"], + ends=True, + warning_message="CSV not found!", + index_filter=None, +): + """ + Displays a CSV as a Streamlit dataframe. + + Parameters + ---------- + directory : str + Folder in which to look for the CSV. + suffix : str + Suffix (or pattern) used by `find_by_suffix` to pick the file. + title : str + Sub-header shown above the dataframe. + columns : list or None + Which columns to keep. None = show all columns. + ends : bool + Passed straight through to `find_by_suffix`. + warning_message : str + Message shown if the CSV cannot be found. + index_filter : iterable or None + Iterable of row-index values to keep; when None, the full file is shown. + """ + file = find_by_suffix(directory, suffix=suffix, ends=ends) + + if not file: + st.warning(f"⚠️ {warning_message}") + return + + st.subheader(title) + + # ------------------ load ------------------ + df = pd.read_csv(file) + if columns is not None: + df = df[columns] + + # ------------------ filter rows by index ------------------ + if index_filter is not None: + # Ensure we’re working with a set for fast lookup + wanted = set(index_filter) + df = df.loc[df.index.intersection(wanted)] + + # ------------------ display ------------------ + if df.empty: + st.info("No rows match the selected index filter.") + else: + st.dataframe(df) + +def display_wordcloud(directory): + """Helper function to display a word cloud if the file exists.""" + file = find_by_suffix(directory, suffix="wordcloud_", ends=False) + if file: + with st_normal(): # Assuming this is a custom context manager for normalizing UI + st.subheader("Wordcloud") + st.image(Image.open(file), caption="Word Cloud", use_container_width=True) + +def open_explorer_button(directory, key): + """Displays a button to open the file explorer for a given directory.""" + if st.button("Open File Explorer", key=key): + open_file_browser(directory) + +def display_images_in_tabs(tab_objects, png_files): + """Displays PNG images in corresponding tabs.""" + for j, png_file in enumerate(png_files): + with tab_objects[j]: + st.write(f"🖼️ {os.path.basename(png_file)}") + with st_normal(): + st.image(Image.open(png_file), caption=os.path.basename(png_file), use_container_width=True) + +def display_html_selector(tab, html_files, key): + """Displays a select box for choosing an HTML interactive plot.""" + with tab: + if html_files: + options = ["Select a file..."] + [os.path.basename(f) for f in html_files] + selected_html = st.selectbox("Select an interactive plot:", options, key=key) + if selected_html != "Select a file...": + st.session_state.html_figure = next(f for f in html_files if os.path.basename(f) == selected_html) + +def display_csv_in_tab(tab, directory, suffix, warning_message, ends): + """Displays a CSV file if found, otherwise shows a warning.""" + with tab: + file = find_by_suffix(directory, suffix=suffix, ends=ends) + if file: + st.dataframe(pd.read_csv(file)) + else: + st.warning(f"⚠️ {warning_message}") + +def display_view_type_setting(configuration=None): + options = ["Document Analysis", "Link Prediction"] + index = 0 + if configuration is not None: + index = options.index(configuration["VIEW_TYPE"]) + st.session_state.view_type = st.selectbox("View Type:", options=options, index=index) + +def display_file_node_settings(configuration=None): + text_input_columns = st.columns(2) + + if configuration is not None: + document_file_name_suffix = configuration.get('DOCUMENT_FILE_START_WITH') or "best_50_docs_in_" + root_name = configuration.get('ROOT_NAME') or "*" + + else: + document_file_name_suffix = "best_50_docs_in_" + root_name = "*" + + with text_input_columns[0]: + st.session_state.document_file_name_suffix = st.text_input("Document File Start With:", document_file_name_suffix) + with text_input_columns[1]: + st.session_state.root_name = st.text_input("Root Name:", root_name) + +def display_attribute_text_settings(configuration=None): + attribute_text_columns = st.columns(3) + + if configuration is not None: + TEXT_COLUMN = configuration.get('TEXT_COLUMN') + ATTRIBUTE_COLUMN = configuration.get('ATTRIBUTE_COLUMN') + + if TEXT_COLUMN is not None: + text_column_index = st.session_state.available_columns.index(TEXT_COLUMN) + if ATTRIBUTE_COLUMN is not None: + attribute_column_index = st.session_state.available_columns.index(ATTRIBUTE_COLUMN) + + attribute_column_split = configuration.get('ATTRIBUTE_COLUMN_SPLIT_BY') or ", " + + else: + text_column_index = 0 + attribute_column_index = 0 + attribute_column_split = ", " + + with attribute_text_columns[0]: + st.session_state.text_column = st.selectbox("Text Column:", options=st.session_state.available_columns, index=text_column_index) + with attribute_text_columns[1]: + st.session_state.attribute_column = st.selectbox("Attribute Column:", options=st.session_state.available_columns, index=attribute_column_index) + with attribute_text_columns[2]: + st.session_state.attribute_column_split = st.text_input("Attribute Column Split by:", attribute_column_split) diff --git a/TELF/applications/Lynx/frontend/pages/helpers/html_contents.py b/TELF/applications/Lynx/frontend/pages/helpers/html_contents.py new file mode 100644 index 00000000..dcd73999 --- /dev/null +++ b/TELF/applications/Lynx/frontend/pages/helpers/html_contents.py @@ -0,0 +1,238 @@ +import pycountry +import matplotlib.cm as cm +import hashlib +from nltk.stem import PorterStemmer +stemmer = PorterStemmer() + +DARK_COLOR = "#3b3b3b" +LIGHT_COLOR = "#E1E1E1" + +def get_color_for_item(item, settings): + stem = stemmer.stem(item.lower()) + + # Handle no-color setting using visibility mode + if settings.get("contents_coloring") == "No Color": + visibility = settings.get("selected_visibility", "Light") + return DARK_COLOR if visibility == "Dark" else LIGHT_COLOR + + # Otherwise, generate color from colormap + hash_val = int(hashlib.sha256(stem.encode('utf-8')).hexdigest(), 16) + norm_index = (hash_val % (10**8)) / (10**8) + + cmap = cm.get_cmap(settings["contents_coloring"]) + rgba = cmap(norm_index) + rgb = tuple(int(x * 255) for x in rgba[:3]) + + return f'rgb{rgb}' + +def get_flag_for_country(name): + try: + country = pycountry.countries.get(name=name) + if not country or len(country.alpha_2) != 2: + return name + code = country.alpha_2.upper() + flag = chr(127397 + ord(code[0])) + chr(127397 + ord(code[1])) + return f"{flag} {name}" + except: + return name + +def render_tree_helper(data, data_map, settings): + html = "" + return html + + + +def render_tree(tree_data, data_map, settings): + # Set font color based on theme + font_color = "#000000" if settings["selected_visibility"] == "Dark" else "#ffffff" + + html_content = f""" + + + + + + + +
+ + +
+ + {render_tree_helper( + data=tree_data, + data_map=data_map, + settings=settings, + )} + + + + """ + return html_content + diff --git a/TELF/applications/Lynx/frontend/pages/helpers/load_project_data.py b/TELF/applications/Lynx/frontend/pages/helpers/load_project_data.py new file mode 100644 index 00000000..a39f7643 --- /dev/null +++ b/TELF/applications/Lynx/frontend/pages/helpers/load_project_data.py @@ -0,0 +1,426 @@ +import streamlit as st +import os +import re +from pages.helpers.utils import (split_and_flatten_list, load_csv_file_items, + extract_affiliation_info, get_top_words, + find_folder_by_prefix, file_exists_in_path, + get_token_index_map) +from pykeen.triples import TriplesFactory +import torch +import pickle +import json + +@st.cache_resource +def load_link_data(path): + + # PYKEEN TYPE + if find_folder_by_prefix(path, "pykeen_model") and file_exists_in_path(path, "data.p"): + # Under the project must have pykeen_model folder (results.save_to_directory()), and data.p + # data.p is a dictinary of {rows:list, columns:list, attributes:dict, triples:df} + # attributes:dict keys are the node names from rows and cols, and value of the attributes:dict are dictionary where key is attribute type and value is the attribute value + # Example attributes:dict = {"Node 1":{"country":"USA"}} + # triples:df is a dataframe with columns head, relation, and tail where head is from rows, and tail is from columns. + model = torch.load(os.path.join(path, "pykeen_model", "trained_model.pkl"), weights_only=False, map_location=torch.device('cpu')) + training_triples_factory = TriplesFactory.from_path_binary((os.path.join(path, "pykeen_model", "training_triples"))) + data = pickle.load(open(os.path.join(path, "data.p"), "rb")) + return {"data":data, "training_triples_factory":training_triples_factory, "model":model, "type":"pykeen"} + + # TELF TYPE + elif file_exists_in_path(path, "telf.p"): + # Under the project must have telf.p + # telf.p is a dictionary of {rows:list, columns:list, attributes:dict, triples:df, Xtilda:np.ndarray, MASK:np.ndarray} + # attributes:dict keys are the node names from rows and cols, and value of the attributes:dict are dictionary where key is attribute type and value is the attribute value + # Example attributes:dict = {"Node 1":{"country":"USA"}} + # triples:df is a dataframe with columns head, relation, and tail where head is from rows, and tail is from columns. + data = pickle.load(open(os.path.join(path, "telf.p"), "rb")) + return {"data":data, "type":"telf"} + + # NOT A Link Prediction PROJECT + else: + # todo + return None + +@st.cache_data +def load_document_analysis_data(path, text_column, document_file_name_suffix, attribute_column, attribute_column_split, root_name): + graph = build_graph_from_directory(path, root_name) + tree = build_tree(graph, root_name) + documents_data, tree_map = extract_data_from_graph(graph, path, + text_column, + document_file_name_suffix, + attribute_column, attribute_column_split) + + return tree, tree_map, documents_data + +def extract_data_from_graph(graph, path, text_column, document_file_name_suffix, attribute_column, attribute_column_split): + """Extracts unigrams, authors, affiliations, and countries from the graph data.""" + documents_data = { + "all_words": {}, + "all_words_denovo":{}, + "all_authors": {}, + "all_affiliations": {}, + "all_countries": {}, + "all_text":{}, + "all_attributes":{} + } + tree_map = {} + + for node in graph: + node_label = f"{node['name']}-{node['label']} ({node['document_count']})" + + # Extract unigrams + try: + unigrams = load_csv_file_items( + path=node["path"], + suffix="_bow_unigrams.csv", + ends=True, + column="word" + ) + except: + unigrams = [] + + if unigrams is not None: + documents_data["all_words"][node_label] = list(unigrams) + + # Extract authors + try: + authors_raw, authors_index_map = split_and_flatten_list(load_csv_file_items( + path=node["path"], + suffix=document_file_name_suffix, + ends=False, + column="author_ids" + ), use_index_map=True) + authors = list(set(authors_raw)) + except: + authors = [] + + if authors is not None: + documents_data["all_authors"][node_label] = authors + + # Extract affiliations and countries + try: + affiliations = load_csv_file_items( + path=node["path"], + suffix=document_file_name_suffix, + ends=False, + column="affiliations" + ) + organizations, countries, org_index_map, country_index_map = extract_affiliation_info(affiliations, use_index_map=True) + except: + organizations = [] + countries = [] + + documents_data["all_affiliations"][node_label] = organizations + documents_data["all_countries"][node_label] = countries + + # Extract denovo words + try: + all_text = load_csv_file_items( + path=node["path"], + suffix=document_file_name_suffix, + ends=False, + column=str(text_column) + ) + except: + all_text = [] + + documents_data["all_text"][node_label] = all_text + denovo_unigrams = get_top_words(all_text, top_n=50, n_gram=1, verbose=False, filename=None)["word"] + denovo_unigrams_index_map = get_token_index_map(all_text, list(denovo_unigrams)) + documents_data["all_words_denovo"][node_label] = list(denovo_unigrams) + + # Extract attributes + try: + all_attributes, attributes_index_map = split_and_flatten_list(load_csv_file_items( + path=node["path"], + suffix=document_file_name_suffix, + ends=False, + column=str(attribute_column) + ), split_by=attribute_column_split, use_index_map=True) + except: + all_attributes = [] + documents_data["all_attributes"][node_label] = all_attributes + + # Store data in tree_map + tree_map[node["name"]] = { + "path": node["path"], + "label": node_label, + "unigrams": unigrams, + "denovo_unigrams":denovo_unigrams, + "denovo_unigrams_index_map":denovo_unigrams_index_map, + "author": authors, + "author_index_map": authors_index_map, + "affiliation": organizations, + "affiliation_index_map": org_index_map, + "country": countries, + "country_index_map": country_index_map, + "all_text": all_text, + "attributes":all_attributes, + "attributes_index_map":attributes_index_map, + } + + return documents_data, tree_map + + +def build_tree(nodes, root_name): + node_dict = {node["name"]: node for node in nodes} + tree = [] + + def add_children(node_name): + node = node_dict[node_name] + label = f"{node['name']}-{node['label']} ({node['document_count']})" + + return { + "label": label, + "value": node_name, + "children": [add_children(child) for child in node["children"]] + } + + # build from root node(s) + for node in nodes: + if node["name"] == root_name: # or any other condition for your root + tree.append(add_children(node["name"])) + return tree + +def parse_topic_folder_name(folder_name, path=None): + """ + Extracts topic number, label, and document count from folder name. + + Expected format: topic_number-label_with_underscores-documents_count-documents + Example: '3-label_of_the_topic_9-documents' -> ('3', 'Label of the topic', '9') + + Args: + folder_name (str): Folder name formatted as: -_-documents + + Returns: + tuple: (str, str, str) -> (topic_number, cleaned_label, document_count) + or (None, None, None) if parsing fails. + """ + match = re.match(r'(\d+)-([^-]+)_(\d+)-documents', folder_name) + if match: + topic_number = match.group(1) # Extract topic number + label = match.group(2).replace("_", " ").strip() # Replace underscores with spaces + document_count = match.group(3) # Extract document count + return topic_number, label, document_count + elif folder_name.isdigit(): + labels = load_csv_file_items(path, suffix="cluster_summaries", ends=False, column="label") + if path is None or labels is None: + return folder_name, "Topic", "Unknown" + else: + return folder_name, labels[int(folder_name)], "Unknown" + + return None, None, None + + +def map_folder_to_logical_name(folder_name: str, root_name) -> str: + """ + Convert a physical folder name (like '*_0', '*_1_2', or '0_1') + to a 'logical' node name. + + Examples: + map_folder_to_logical_name("*") -> "*" + map_folder_to_logical_name("*_0") -> "0" + map_folder_to_logical_name("*_1_5") -> "1_5" + map_folder_to_logical_name("1_2") -> "1_2" + map_folder_to_logical_name("0") -> "0" + + The idea is to strip a leading '*_' if present, because + '*_0' means "the node for root topic 0." + """ + if folder_name == root_name: + return root_name + # If it starts with "*_", remove it + if folder_name.startswith(f"{root_name}_"): + return folder_name[len(root_name)+1:] # remove the '*_' prefix + return folder_name + + +def get_logical_parent_name(logical_name: str, root_name): + """ + Given a logical name (like '0', '1_2', '1_2_3'), return its parent. + + Rules: + - If the logical name is "*", it's the root (no parent). + - If there's only one chunk and it's not "*", parent is "*". + - Otherwise, drop the last chunk. + + Examples: + get_logical_parent_name("*") -> None + get_logical_parent_name("0") -> "*" + get_logical_parent_name("1_5") -> "1" + get_logical_parent_name("1_5_0") -> "1_5" + """ + + if logical_name == root_name: + return None # root + + parts = logical_name.split("_") + if len(parts) == 1: + # single chunk (e.g. "0", "1") => parent is "*" + return root_name + else: + # multiple chunks => drop the last chunk + return "_".join(parts[:-1]) + + +def build_graph_from_directory(root_path, root_name): + """ + Parses the hierarchical directory structure to build a list of nodes. + + Each folder in `depth_i` is considered a "container folder". + Inside that folder, we expect subfolders that match the topic pattern: + -_-documents + + We map each container folder to a 'logical' name, and + each topic subfolder becomes a child node of that container folder. + + Args: + root_path (str): Path to the root directory. + + Returns: + list: A list of dictionaries representing the nodes in the graph. + """ + # Dictionary of node_name -> node_data + nodes = {} + + # Ensure we have a root node named "*", or root_name + if root_name not in nodes: + nodes[root_name] = { + "name": root_name, + "label": "Root", + "parent": None, + "children": [], + "document_count": None, + "path": root_path + } + + # Iterate through depth_* folders + for depth_folder in sorted(os.listdir(root_path)): + depth_path = os.path.join(root_path, depth_folder) + if not os.path.isdir(depth_path) or not depth_folder.startswith("depth_"): + continue # ignore non-depth folders + # Iterate through all container folders at this depth + for physical_folder_name in sorted(os.listdir(depth_path)): + container_folder_path = os.path.join(depth_path, physical_folder_name) + if not os.path.isdir(container_folder_path): + continue # skip files + + # Map physical folder to a logical node name + container_logical_name = map_folder_to_logical_name(physical_folder_name, root_name) + + # Ensure the container folder node itself exists + if container_logical_name not in nodes: + # figure out its parent + parent_name = get_logical_parent_name(str(container_logical_name), root_name) + + nodes[container_logical_name] = { + "name": container_logical_name, + "label": None, # might not be a "topic" node but a container + "parent": parent_name, + "children": [], + "document_count": None, + "path": container_folder_path + } + # link to parent + if parent_name and parent_name not in nodes: + # create placeholder for parent's node if missing + nodes[parent_name] = { + "name": parent_name, + "label": None, + "parent": get_logical_parent_name(str(parent_name), root_name), + "children": [], + "document_count": None, + "path": None + } + if parent_name: + nodes[parent_name]["children"].append(container_logical_name) + + # Now check all "topic subfolders" in this container + topic_folders = sorted( + f for f in os.listdir(container_folder_path) + if os.path.isdir(os.path.join(container_folder_path, f)) + ) + + for topic_folder in topic_folders: + topic_number, label, document_count = parse_topic_folder_name(topic_folder, container_folder_path) + if topic_number is None: + # Not a recognized topic folder; skip + continue + + # Build the full node name for the topic + if container_logical_name == root_name: + # If container is the global root "*" + # then the node name is just the topic_number + topic_node_name = topic_number + else: + # Otherwise, we append the topic_number to the container's logical name + topic_node_name = f"{container_logical_name}_{topic_number}" + + topic_path = os.path.join(container_folder_path, topic_folder) + + # If this topic node doesn't exist yet, create it + if topic_node_name not in nodes: + nodes[topic_node_name] = { + "name": topic_node_name, + "label": label, + "parent": container_logical_name, # parent is the container folder node + "children": [], + "document_count": document_count, + "path": topic_path + } + + # Add it as a child of the container folder node + if container_logical_name not in nodes: + # container might not have been created yet if it's missing + # but we created it above, so this is just a safety check + nodes[container_logical_name] = { + "name": container_logical_name, + "label": None, + "parent": get_logical_parent_name(str(container_logical_name), root_name), + "children": [], + "document_count": None, + "path": container_folder_path + } + nodes[container_logical_name]["children"].append(topic_node_name) + + # Convert node dictionary to a list + return list(nodes.values()) + +def load_config(file_path): + """ + Loads a JSON configuration file. + + Args: + file_path: The path to the JSON file. + + Returns: + A dictionary containing the configuration data, or None if the file + could not be loaded. + """ + try: + with open(file_path, 'r') as f: + config = json.load(f) + return config + except FileNotFoundError: + st.warning(f"Error: File not found at {file_path}") + return None + except json.JSONDecodeError: + st.warning(f"Error: Invalid JSON format in {file_path}") + return None + +def set_configs(configuration): + """ + Sets Streamlit session state variables from a configuration dictionary, + checking for key existence first. + """ + if "VIEW_TYPE" in configuration: + st.session_state.view_type = configuration["VIEW_TYPE"] + if "DOCUMENT_FILE_START_WITH" in configuration: + st.session_state.document_file_name_suffix = configuration["DOCUMENT_FILE_START_WITH"] + if "ROOT_NAME" in configuration: + st.session_state.root_name = configuration["ROOT_NAME"] + if "TEXT_COLUMN" in configuration: + st.session_state.text_column = configuration["TEXT_COLUMN"] + if "ATTRIBUTE_COLUMN" in configuration: + st.session_state.attribute_column = configuration["ATTRIBUTE_COLUMN"] + if "ATTRIBUTE_COLUMN_SPLIT_BY" in configuration: + st.session_state.attribute_column_split = configuration["ATTRIBUTE_COLUMN_SPLIT_BY"] \ No newline at end of file diff --git a/TELF/applications/Lynx/frontend/pages/helpers/utils.py b/TELF/applications/Lynx/frontend/pages/helpers/utils.py new file mode 100644 index 00000000..3b75e109 --- /dev/null +++ b/TELF/applications/Lynx/frontend/pages/helpers/utils.py @@ -0,0 +1,694 @@ +import streamlit as st +import subprocess +import platform +import os +import numpy as np +import pandas as pd +import ast +from tqdm import tqdm +from collections import defaultdict +import re +import networkx as nx +from pykeen import predict + +def build_coauthorship_network(authors, author_ids): + """ + Build a co-authorship network graph from the dataframe. + Nodes represent authors, edges represent co-authorship occurrences. + """ + G = nx.Graph() + author_pairs = {} + + for authors, ids in zip(authors, author_ids): + authors_list = preprocess_authors([authors], [ids]) + + for i in range(len(authors_list)): + for j in range(i + 1, len(authors_list)): + pair = tuple(sorted([authors_list[i], authors_list[j]])) + if pair in author_pairs: + author_pairs[pair] += 1 + else: + author_pairs[pair] = 1 + + for (author1, author2), count in author_pairs.items(): + G.add_edge(author1, author2, weight=count) + + return G + +@st.cache_data +def get_top_words(documents, + top_n=10, + n_gram=1, + verbose=True, + filename=None) -> pd.DataFrame: + """ + Collects statistics for the top words or n-grams. Returns a table with columns + word, tf, df, df_fraction, and tf_fraction. + + - word column lists the words in the top_n. + - tf is the term-frequency, how many times given word occured in documents. + - df is the document-frequency, in how documents given word occured. + - df_fraction is df / len(documents) + - tf_fraction is tf / (total number of unique tokens or n-grams) + + Parameters + ---------- + documents : list or dict + list or dictionary of documents. + If dictionary, keys are the document IDs, values are the text. + top_n : int, optional + Top n words or n-grams to report. The default is 10. + n_gram : int, optional + 1 is words, or n-grams when > 1. The default is 1. + verbose : bool, optional + Verbosity flag. The default is True. + filename : str, optional + If not one, saves the table to the given location. + + Returns + ------- + pd.DataFrame + Table for the statistics. + + """ + + if isinstance(documents, dict): + documents = list(documents.values()) + + word_stats = defaultdict(lambda: {"tf": 0, "df": 0}) + + for doc in tqdm(documents, disable=not verbose): + tokens = doc.split() + ngrams = zip(*[tokens[i:] for i in range(n_gram)]) + ngrams = [" ".join(ngram) for ngram in ngrams] + + for gram in ngrams: + word_stats[gram]["tf"] += 1 + for gram in set(ngrams): + word_stats[gram]["df"] += 1 + + word_stats = dict(word_stats) + top_words = dict(sorted(word_stats.items(), key=lambda x: x[1]["tf"], reverse=True)[:top_n]) + target_data = {"word": [], "tf": [], "df": [], "df_fraction": [], "tf_fraction": []} + + for word in top_words: + target_data["word"].append(word) + target_data["tf"].append(word_stats[word]["tf"]) + target_data["df"].append(word_stats[word]["df"]) + target_data["df_fraction"].append(word_stats[word]["df"] / len(documents)) + target_data["tf_fraction"].append(word_stats[word]["tf"] / len(word_stats)) + + # put together the results + table = pd.DataFrame.from_dict(target_data) + + if filename: + table.to_csv(filename+".csv", index=False) + + return table + + + +def extract_unique_values(data_key, data): + """Extract unique values from nested lists in session state.""" + return list(set(a for alist in data[data_key].values() for a in alist)) + +def extract_all_affiliation_info(affiliations): + organizations = list() + countries = list() + + for curr_affil in affiliations: + if not curr_affil or curr_affil in ["Nan", "nan", None, np.nan]: + continue # Skip invalid rows + + try: + curr_affil = ast.literal_eval(curr_affil) # Convert string to dictionary + if not isinstance(curr_affil, dict): + continue # Ensure it's a dictionary + + for affil_id, values in curr_affil.items(): + if not isinstance(values, dict): + continue # Ensure values is a dictionary + + # Extract and clean name + name = values.get("name", "").strip() + if not name or name in ["Nan", "nan", None, np.nan]: + continue # Skip empty names + + # Extract and clean country + country = values.get("country", "").strip() + if country and country not in ["Nan", "nan", None, np.nan]: + countries.append(country) + + organizations.append(f"{name} {affil_id}") + + except (ValueError, SyntaxError): + continue # Handle cases where eval fails (e.g., malformed data) + + return list(organizations), list(countries) + +def get_token_index_map(texts, tokens): + index_map = {} + + for idx, text in enumerate(texts): + if not isinstance(text, str): + continue # Skip non-string entries + for token in tokens: + if token in text: + index_map.setdefault(token, []).append(idx) + + return index_map + +def extract_affiliation_info(affiliations, use_index_map=False): + organizations = set() + countries = set() + org_index_map = {} + country_index_map = {} + + for idx, curr_affil in enumerate(affiliations): + if not curr_affil or curr_affil in ["Nan", "nan", None, np.nan]: + continue # Skip invalid rows + + try: + curr_affil = ast.literal_eval(curr_affil) # Convert string to dictionary + if not isinstance(curr_affil, dict): + continue # Ensure it's a dictionary + + for affil_id, values in curr_affil.items(): + if not isinstance(values, dict): + continue # Ensure values is a dictionary + + # Extract and clean name + name = values.get("name", "").strip() + if not name or name in ["Nan", "nan", None, np.nan]: + continue # Skip empty names + + org_key = f"{name} {affil_id}" + organizations.add(org_key) + if use_index_map: + org_index_map.setdefault(org_key, []).append(idx) + + # Extract and clean country + country = values.get("country", "").strip() + if country and country not in ["Nan", "nan", None, np.nan]: + countries.add(country) + if use_index_map: + country_index_map.setdefault(country, []).append(idx) + + except (ValueError, SyntaxError): + continue # Handle cases where eval fails (e.g., malformed data) + + if use_index_map: + return list(organizations), list(countries), org_index_map, country_index_map + else: + return list(organizations), list(countries) + + +def filter_documents( + data_map, + top_n, + selected_words, + token_search_type, + attr_selections, + search_categories, + keywords_search_type, + use_index_map=False, + negative_words=None, +): + """ + Returns: + selected_labels – {doc_label: 1} + selected_indices (optional) – {doc_label: sorted(list_of_indices)} + """ + + selected_labels = {} + selected_indices = {} # populated only when use_index_map=True + + # ------------------------------------------------------------ + # Decide which unigram list is being searched + keywords_search_field = ( + "unigrams" if keywords_search_type == "Keywords" else "denovo_unigrams" + ) + # NOTE: we *always* use denovo_unigrams_index_map for index positions + UNIGRAM_INDEX_MAP_FIELD = "denovo_unigrams_index_map" + + # ------------------------------------------------------------ + for doc_id, doc_val in data_map.items(): + + # ============== 1) KEYWORD-BASED FILTER ================= + keyword_pass = True + keyword_idx_set = set() + + if selected_words or negative_words: # only check if user gave tokens + # Limit to Top-N if needed + if top_n == "All": + doc_unigrams = set(doc_val[keywords_search_field]) + max_allowed_idx = None + else: + doc_unigrams = set(doc_val[keywords_search_field][:top_n]) + max_allowed_idx = top_n - 1 + + # AND / OR logic + if token_search_type == "and": + keyword_pass = all(w in doc_unigrams for w in selected_words) + else: + keyword_pass = any(w in doc_unigrams for w in selected_words) + + if negative_words: # Check if there are words to exclude + keyword_pass = keyword_pass and not any(w in doc_unigrams for w in negative_words) + + if not keyword_pass: + continue + + # Collect indices *only* for DeNovo searches + if use_index_map and keywords_search_type != "Keywords": + unigram_map = doc_val.get(UNIGRAM_INDEX_MAP_FIELD, {}) + + # Collect indices of all negative words (if any) + negative_idx_set = set() + if negative_words: + for neg_word in negative_words: + negative_idx_set.update(unigram_map.get(neg_word, [])) + print(negative_idx_set, unigram_map) + for w in selected_words: + for i in unigram_map.get(w, []): + if (max_allowed_idx is None or i <= max_allowed_idx) and i not in negative_idx_set: + keyword_idx_set.add(i) + + # ============== 2) ATTRIBUTE-BASED FILTERS ============== + attr_pass = True + # We’ll build one union set per attribute category; those unions + # are later intersected across categories (and with keywords). + per_category_sets = [] + + for label, info in search_categories.items(): + attr_key = info["key"] # e.g. 'country' + chosen_vals = attr_selections.get(f"selected_{attr_key}", []) + logic = attr_selections.get(f"{attr_key}_search_type", "and") + + if not chosen_vals: + continue # no filter for this attr + + doc_attr_set = set(doc_val.get(attr_key, [])) + + if logic == "and": + if not set(chosen_vals).issubset(doc_attr_set): + attr_pass = False + break + else: + if not doc_attr_set.intersection(chosen_vals): + attr_pass = False + break + + # Add index positions for this attribute category + if use_index_map: + attr_map = doc_val.get(f"{attr_key}_index_map", {}) + + if logic == "and": + # intersection across every selected value + idx_set = None + for v in chosen_vals: + v_idx = set(attr_map.get(v, [])) + idx_set = v_idx if idx_set is None else idx_set & v_idx + else: # "or" + # union across every selected value + idx_set = set() + for v in chosen_vals: + idx_set.update(attr_map.get(v, [])) + + per_category_sets.append(idx_set) + + if not attr_pass: + continue + + # ============== 3) CONSOLIDATE INDICES ================== + if use_index_map: + # Start with keyword indices (may be empty) + if keyword_idx_set: + consolidated = keyword_idx_set.copy() + else: + consolidated = None # will adopt first set + + # Intersect with every attribute category’s index set + for s in per_category_sets: + if consolidated is None: + consolidated = s.copy() + else: + consolidated &= s + + # If we never gathered any sets, consolidated stays None + final_idx_list = sorted(consolidated) if consolidated else None + selected_indices[doc_val["label"]] = final_idx_list + + # ============== 4) RECORD DOCUMENT ====================== + selected_labels[doc_val["label"]] = 1 + + # ------------------------------------------------------------ + if use_index_map: + return selected_labels, selected_indices + + return selected_labels, None + + +def preprocess_authors(authors_col, author_ids_col): + """ + Process author and author_id columns to ensure correct matching. + Handles None, NaN, and "None" values. + """ + processed_authors = [] + + for authors, ids in zip(authors_col, author_ids_col): + if pd.isna(authors) or authors in ["None", "none", ""]: + authors_list = [] + else: + authors_list = authors.split(";") + + if pd.isna(ids) or ids in ["None", "none", ""]: + ids_list = [] + else: + ids_list = ids.split(";") + + matched_authors = [ + f"{name.strip()} ({id_.strip()})" if name.strip() else f"( {id_.strip()} )" + for name, id_ in zip(authors_list, ids_list) + ] + processed_authors.extend(matched_authors) + + return processed_authors + +def split_and_flatten_list(lst, split_by=";", use_index_map=False): + result = [] + index_map = {} + + for idx, item in enumerate(lst): + if isinstance(item, float) and np.isnan(item): # Handling NaN (float type) + continue + elif isinstance(item, str) and item.lower() == "nan": # Handling "nan" as a string + continue + else: + parts = [x.strip() for x in str(item).split(split_by) if x.strip()] + result.extend(parts) + if use_index_map: + for part in parts: + index_map.setdefault(part, []).append(idx) + + if use_index_map: + return result, index_map + else: + return result + +def find_unique_special_chars(column): + """ + Finds all unique special characters in a given pandas DataFrame column. + + :param column: A pandas Series (column) containing text data + :return: A set of unique special characters found in the column + """ + special_chars = set() + + # Regular expression to match special characters (excluding letters, numbers, and spaces) + pattern = re.compile(r'[^a-zA-Z0-9\s]') + + # Iterate through the column + for text in column.dropna(): # Drop NaN values to avoid errors + special_chars.update(pattern.findall(str(text))) # Extract and add unique special chars + + return special_chars + +def load_csv_file_items(path, suffix, ends, column): + file = find_by_suffix(directory=path, suffix=suffix, ends=ends) + if file: + if isinstance(column, list): + items = np.array(pd.read_csv(file)[column]) + else: + items = np.array(pd.read_csv(file)[column].tolist()) + else: + items = np.array([]) + return items + +def find_files_by_extensions(directory, extensions=("html", "png")): + files = {ext: [] for ext in extensions} + if os.path.exists(directory) and os.path.isdir(directory): + for file in os.listdir(directory): + for ext in extensions: + if file.endswith(f".{ext}"): + files[ext].append(os.path.join(directory, file)) + return files + +def file_exists_in_path(path: str, filename: str) -> bool: + """ + Checks if a file with the given filename exists in the specified path. + + :param path: The directory path to search in. + :param filename: The name of the file to look for. + :return: True if the file exists, False otherwise. + """ + for root, _, files in os.walk(path): + if filename in files: + return True + return False + +def find_folder_by_prefix(directory, prefix): + for item in os.listdir(directory): + item_path = os.path.join(directory, item) + if os.path.isdir(item_path) and item.startswith(prefix): + return item # Returns the first matching folder found + return None # Returns None if no folder matches + +def find_by_suffix(directory, suffix="_bow_bigrams.csv", ends=True): + if os.path.exists(directory) and os.path.isdir(directory): + for file in os.listdir(directory): + if (ends and file.endswith(suffix)) or (not ends and file.startswith(suffix)): + return os.path.join(directory, file) + return None + +@st.cache_data +def extract_attribute_lists(attributes_dict): + organized_attributes = {} + for node_name, attributes in attributes_dict.items(): + for attribute_type, attribute in attributes.items(): + if attribute_type not in organized_attributes: + organized_attributes[attribute_type] = [attribute] + else: + organized_attributes[attribute_type].append(attribute) + return organized_attributes + +@st.cache_data +def extract_link_attributes_df(rows, cols, attributes_dict): + all_unique_nodes = list(set(rows + cols)) + df_dict = {"node":[]} + unique_attribute_types = [] + # collect all potential attributes + for _, attributes in attributes_dict.items(): + for attribute_type, _ in attributes.items(): + if attribute_type not in df_dict: + df_dict[attribute_type] = [] + unique_attribute_types.append(attribute_type) + + # form dataframe + for node in all_unique_nodes: + df_dict["node"].append(node) + for attribute_type in unique_attribute_types: + if node in attributes_dict: + if attribute_type in attributes_dict[node]: + df_dict[attribute_type].append(attributes_dict[node][attribute_type]) + else: + df_dict[attribute_type].append("Unknown") + else: + print(node) + df_dict[attribute_type].append("Unknown") + + return pd.DataFrame.from_dict(df_dict) + +def open_file_browser(path): + if platform.system() == "Windows": + os.startfile(path) + elif platform.system() == "Darwin": + subprocess.run(["open", path]) + elif platform.system() == "Linux": + subprocess.run(["xdg-open", path]) + +def prune_subtree(node, allowed_labels, root_value="*"): + """ + Prune the given 'node' so that: + - If node's label is in allowed_labels (or node is root), + we keep node (plus its pruned children). + - Otherwise, bubble up any pruned children to the node's parent. + - If no child is valid, the node is removed (None). + + Returns: + - A single node dict if we keep the node, + - A list of node dicts if we remove the node but keep its children, + - None if everything under this node is removed. + """ + # 1) Recursively prune children first (post-order) + new_children = [] + for child in node["children"]: + result = prune_subtree(child, allowed_labels, root_value) + if result is None: + # child and its subtree are all invalid => skip + continue + if isinstance(result, list): + # child was removed but has valid children => bubble them up + new_children.extend(result) + else: + # child is a single valid pruned node + new_children.append(result) + + # 2) Decide if we keep 'node' + is_root = (node["value"] == root_value) # e.g., "*" + is_allowed_label = (node.get("label") in allowed_labels) + + # Always keep the root, OR keep if label is allowed + if is_root or is_allowed_label: + # We keep this node, with pruned children + pruned_node = { + **node, # copy existing keys if you want + "children": new_children + } + return pruned_node + else: + # Node label not allowed => bubble its children up + if new_children: + return new_children # Return a list so parent can attach them + return None # No valid children => entire subtree removed + +def prune_tree(root_node, allowed_labels, root_value="*"): + """ + Prune the entire tree starting from 'root_node'. + + If root is removed (not allowed AND no valid children), + we might end up with None or a list of top-level nodes. + Usually, if root_value == '*', we keep the root no matter what. + + Returns: + - A single pruned root node (typical case), + - A list of nodes (if the root was removed or bubbled up children), + - Or None if everything is pruned. + """ + result = prune_subtree(root_node, allowed_labels, root_value) + if result is None: + # Entire tree pruned away + return None + if isinstance(result, list): + # We ended up with a "forest" (multiple siblings at top level) + return result + # A single pruned root node + return result + +def get_attributes_from_df(df, column, attribute_dict): + nodes = df[column] + attributes_extracted = {} + unique_attribute_types = [] + + for node_name, attributes in attribute_dict.items(): + for attribute_type, value in attributes.items(): + if attribute_type not in attributes_extracted: + attributes_extracted[attribute_type] = [] + unique_attribute_types.append(attribute_type) + + for node in nodes: + for attribute_type in unique_attribute_types: + if node not in attribute_dict: + attributes_extracted[attribute_type].append("Unknown") + else: + if attribute_type not in attribute_dict[node]: + attributes_extracted[attribute_type].append("Unknown") + else: + attributes_extracted[attribute_type].append(attribute_dict[node][attribute_type]) + + return attributes_extracted + + +def predict_links_pykeen(model, + training_triples_factory, + prediction_direction, + top_n_prediction, + target_node, + relation): + + if prediction_direction == "Tail": + res = predict.predict_target( + model=model, + head=target_node, + relation=relation, + triples_factory=training_triples_factory).filter_triples(training_triples_factory).df.head(top_n_prediction) + + elif prediction_direction == "Head": + res = predict.predict_target( + model=model, + tail=target_node, + relation=relation, + triples_factory=training_triples_factory).filter_triples(training_triples_factory).df.head(top_n_prediction) + + return res + +def predict_links_telf( + Xtilda, + MASK, + rows, + cols, + prediction_direction, + top_n_prediction, + target_node +): + """ + Returns top predictions in DataFrame format. + + Parameters: + Xtilda (np.array): Score matrix of shape (len(rows), len(cols)) + MASK (np.array): Mask matrix of shape (len(rows), len(cols)) with 0s for unknowns and 1s for knowns. + rows (list of str): Row labels. + cols (list of str): Column labels. + prediction_direction (str): Either "head" or "tail", determines whether target_node is in rows or cols. + top_n_prediction (int): Number of top predictions to return. + target_node (str): Node to make predictions for. + + Returns: + pd.DataFrame: DataFrame with columns ['head_label', 'relation', 'tail_label', 'score']. + """ + + # Identify the index based on prediction direction + if prediction_direction == "Head": + if target_node not in cols: + raise ValueError(f"Target node {target_node} not found in columns.") + target_index = cols.index(target_node) + scores = Xtilda[:, target_index] # Get scores for this column + mask = MASK[:, target_index] # Get mask for this column + head_labels = rows # Rows are head labels + relation_label = "exist" + tail_labels = [target_node] * len(rows) # Tail is fixed + + elif prediction_direction == "Tail": + if target_node not in rows: + raise ValueError(f"Target node {target_node} not found in rows.") + target_index = rows.index(target_node) + scores = Xtilda[target_index, :] # Get scores for this row + mask = MASK[target_index, :] # Get mask for this row + head_labels = [target_node] * len(cols) # Head is fixed + relation_label = "exist" + tail_labels = cols # Columns are tail labels + + else: + raise ValueError("prediction_direction must be either 'head' or 'tail'.") + + # Filter to unknowns using the mask + unknown_indices = np.where(mask == 0)[0] + filtered_scores = scores[unknown_indices] + filtered_head_labels = [head_labels[i] for i in unknown_indices] + filtered_tail_labels = [tail_labels[i] for i in unknown_indices] + + # Get top predictions + top_indices = np.argsort(filtered_scores)[::-1][:top_n_prediction] # Sort descending + top_head_labels = [filtered_head_labels[i] for i in top_indices] + top_tail_labels = [filtered_tail_labels[i] for i in top_indices] + top_scores = [filtered_scores[i] for i in top_indices] + + # Construct result DataFrame + df = pd.DataFrame({ + "head_label": top_head_labels, + "relation": [relation_label] * len(top_scores), + "tail_label": top_tail_labels, + "score": top_scores + }) + + return df \ No newline at end of file diff --git a/TELF/applications/Lynx/frontend/pages/html_viewer.py b/TELF/applications/Lynx/frontend/pages/html_viewer.py new file mode 100644 index 00000000..babd6087 --- /dev/null +++ b/TELF/applications/Lynx/frontend/pages/html_viewer.py @@ -0,0 +1,54 @@ +import streamlit as st +import os + +# Get file path from query parameters +if "html_figure" not in st.session_state: + st.warning("⚠️ No HTML figure has been loaded.") + st.stop() # This prevents the rest of the page from rendering + +file_path = st.session_state.html_figure + +if not file_path or not os.path.exists(file_path): + st.error("Invalid or missing HTML file.") +else: + #st.title(file_path) + st.markdown( + """ + + """, + unsafe_allow_html=True + ) + + + # Read and display HTML content + with open(file_path, "r", encoding="utf-8") as f: + html_content = f.read() + + st.components.v1.html( + html_content, + height=0, # Handled by CSS + scrolling=True # Prevent unnecessary scrolling + ) \ No newline at end of file diff --git a/TELF/applications/Lynx/frontend/pages/link_view.py b/TELF/applications/Lynx/frontend/pages/link_view.py new file mode 100644 index 00000000..67d72521 --- /dev/null +++ b/TELF/applications/Lynx/frontend/pages/link_view.py @@ -0,0 +1,127 @@ +import streamlit as st +from pages.helpers.utils import (extract_attribute_lists, + extract_link_attributes_df, + predict_links_pykeen, + get_attributes_from_df, + predict_links_telf) +from pages.helpers.displays import (display_attributes, + display_link_node_search) +import pandas as pd +st.set_page_config(layout="wide") +if ("project_loaded" in st.session_state and st.session_state.project_loaded == False) or "data" not in st.session_state: + st.warning("⚠️ Data has not been loaded yet. Please load the data before accessing this page.") + st.stop() + +if st.session_state.view_type != "Link Prediction": + st.warning("⚠️ Link Prediction mode is not selected.") + st.stop() + +st.header(st.session_state.selected_project) +st.markdown(f"**{st.session_state.data['type'].upper()} project**") + +# +### ATTRIBUTES +# +st.subheader("Attributes Display") +attribute_lists = extract_attribute_lists(st.session_state.data["data"]["attributes"]) + +with st.expander("All Attribute Stats"): + attribute_lists_tabs = st.tabs(attribute_lists.keys()) + for idx, (attribute_name, all_attributes) in enumerate(attribute_lists.items()): + with attribute_lists_tabs[idx]: + display_attributes(all_attributes, key=f"link_attribute_{attribute_name}") + +st.divider() + +# +### NODES +# +st.subheader("Node Search") +link_attributes_df = extract_link_attributes_df( + st.session_state.data["data"]["rows"], + st.session_state.data["data"]["cols"], + st.session_state.data["data"]["attributes"] +) +display_link_node_search(link_attributes_df) +st.divider() + +# +### PREDICTIONS +# +st.subheader("Predict Link") +prediction_settings_columns = st.columns(3) +with prediction_settings_columns[0]: + prediction_direction = st.selectbox("Prediction Type", options=["Tail", "Head"]) +with prediction_settings_columns[1]: + top_n_prediction = st.number_input("Top n Predictions", min_value=1, value=10) +with prediction_settings_columns[2]: + relation = st.selectbox("Relation", + options=st.session_state.data["data"]["triples"]["relation"].unique(), + disabled=st.session_state.data['type'] != "pykeen") + +if prediction_direction == "Tail": + prediction_options = st.session_state.data["data"]["rows"] +elif prediction_direction == "Head": + prediction_options = st.session_state.data["data"]["cols"] +target_node = st.selectbox(f"Select {prediction_direction} Node", options=prediction_options) + +if st.session_state.data['type'] == "pykeen": + predictions_df = predict_links_pykeen( + model=st.session_state.data["model"], + training_triples_factory=st.session_state.data["training_triples_factory"], + prediction_direction=prediction_direction, + top_n_prediction=top_n_prediction, + target_node=target_node, + relation=relation + ) + +elif st.session_state.data['type'] == "telf": + predictions_df = predict_links_telf( + Xtilda=st.session_state.data["data"]["Xtilda"], + MASK=st.session_state.data["data"]["MASK"], + rows=st.session_state.data["data"]["rows"], + cols=st.session_state.data["data"]["cols"], + prediction_direction=prediction_direction, + top_n_prediction=top_n_prediction, + target_node=target_node, + ) + +st.markdown("**Predictions:**") +st.dataframe(predictions_df, use_container_width=True) +prediction_attributes = get_attributes_from_df( + predictions_df, + column=f"{prediction_direction.lower()}_label", + attribute_dict=st.session_state.data["data"]["attributes"]) + +with st.expander("Prediction Attribute Stats"): + prediction_attribute_tabs = st.tabs(prediction_attributes.keys()) + for idx, (attribute_name, all_attributes) in enumerate(prediction_attributes.items()): + with prediction_attribute_tabs[idx]: + display_attributes(all_attributes, key=f"link_prediction_attribute_{attribute_name}") + +st.markdown("**Known relations:**") +if prediction_direction == "Head": + filter_known_df = st.session_state.data["data"]["triples"][st.session_state.data["data"]["triples"]["tail"] == target_node] + st.dataframe( + filter_known_df, + use_container_width=True + ) +elif prediction_direction == "Tail": + filter_known_df = st.session_state.data["data"]["triples"][st.session_state.data["data"]["triples"]["head"] == target_node] + st.dataframe( + filter_known_df, + use_container_width=True + ) + +prediction_known_attributes = get_attributes_from_df( + filter_known_df, + column=f"{prediction_direction.lower()}", + attribute_dict=st.session_state.data["data"]["attributes"]) + +with st.expander("Known Attribute Stats"): + prediction_known_attribute_tabs = st.tabs(prediction_known_attributes.keys()) + for idx, (attribute_name, all_attributes) in enumerate(prediction_known_attributes.items()): + with prediction_known_attribute_tabs[idx]: + display_attributes(all_attributes, key=f"link_prediction_known_attribute_{attribute_name}") + + \ No newline at end of file diff --git a/TELF/applications/Lynx/frontend/pages/load_project.py b/TELF/applications/Lynx/frontend/pages/load_project.py new file mode 100644 index 00000000..e1164ec4 --- /dev/null +++ b/TELF/applications/Lynx/frontend/pages/load_project.py @@ -0,0 +1,151 @@ +import streamlit as st +import os +import pandas as pd + +from pages.helpers.load_project_data import (load_document_analysis_data, load_link_data, + load_config, set_configs) +from pages.helpers.utils import find_by_suffix, find_folder_by_prefix +from pages.helpers.displays import (display_view_type_setting, display_file_node_settings, + display_attribute_text_settings) + +# Set the path of the Projects folder +PROJECTS_FOLDER = "projects" # Change this to your actual projects folder path + +# Function to get all project folders +def get_project_folders(root_path): + if not os.path.exists(root_path): + return [] + return sorted([folder for folder in os.listdir(root_path) if os.path.isdir(os.path.join(root_path, folder))]) + +# Get list of projects +project_folders = get_project_folders(PROJECTS_FOLDER) + +st.title("📂 Select a Project") + +st.divider() + +if not project_folders: + st.warning("No projects found in the Projects folder.") +else: + + # Use selectbox for a modern dropdown (scrollable & searchable) + selected_project = st.selectbox("Projects:", project_folders) + + + # Get full path of the selected project + selected_project_path = os.path.join(PROJECTS_FOLDER, selected_project) + + # + # SETTINGS + # + if selected_project_path: + st.divider() + st.subheader("Settings") + + settings_file = find_by_suffix(os.path.join(selected_project_path), suffix="settings", ends=False) + if settings_file: + available_settings_setup = ["Manual", "Automatic"] + settings_setup_index = 1 + else: + available_settings_setup = ["Manual"] + settings_setup_index = 0 + + selected_settings_setup = st.radio( + "**Settings Setup:**", + available_settings_setup, + index=settings_setup_index, + horizontal=True, + ) + + if "configuration" in st.session_state: + configuration = st.session_state.configuration + if configuration["selected_project_path"] != selected_project_path: + configuration = None + del st.session_state.configuration + else: + configuration = None + + if selected_settings_setup == "Automatic": + configuration = load_config(settings_file) + configuration["selected_project_path"] = selected_project_path + st.session_state.configuration = configuration + set_configs(configuration) + with st.expander(f"Loaded Settings:"): + st.table(configuration) + + if selected_settings_setup == "Manual": + display_view_type_setting(configuration=configuration) + if "view_type" in st.session_state and st.session_state.view_type == "Document Analysis": + display_file_node_settings(configuration=configuration) + + # + # DO LOADING OF THE PROJECT + # + if "view_type" in st.session_state and st.session_state.view_type == "Document Analysis": + + try: + st.session_state.folder_name = find_folder_by_prefix(os.path.join(selected_project_path, "depth_0", st.session_state.root_name), prefix="0") + except: + st.warning(f"⚠️ Can't find the directory at {os.path.join(selected_project_path, 'depth_0', st.session_state.root_name)}") + st.stop() + + file = find_by_suffix(os.path.join(selected_project_path, "depth_0", st.session_state.root_name, st.session_state.folder_name), suffix=st.session_state.document_file_name_suffix, ends=False) + + if file is None: + st.warning(f"⚠️ Document files with suffix **```{st.session_state.document_file_name_suffix}```** does not exist at path **```{os.path.join(selected_project_path, 'depth_0', st.session_state.root_name, folder_name)}```**!") + st.stop() + + st.session_state.available_columns = pd.read_csv(file).columns.to_list() + if selected_settings_setup == "Manual": + display_attribute_text_settings(configuration=configuration) + else: + if st.session_state.text_column not in st.session_state.available_columns: + st.warning(f"The text column {st.session_state.text_column} not in the available columns!") + + if st.session_state.attribute_column not in st.session_state.available_columns: + st.warning(f"The attribute column {st.session_state.attribute_column} not in the available columns!") + + with st.expander(f"Available Columns in `{file}`:"): + st.write(st.session_state.available_columns) + + if st.session_state.text_column not in st.session_state.available_columns: + st.warning(f"⚠️ Text column {st.session_state.text_column} does not exist!") + if st.session_state.attribute_column not in st.session_state.available_columns: + st.warning(f"⚠️ Attribute column {st.session_state.attribute_column} does not exist!") + + + with st.spinner(text="In progress...", show_time=False): + + st.session_state.data, st.session_state.data_map, st.session_state.documents_data = load_document_analysis_data( + selected_project_path, + st.session_state.text_column, + st.session_state.document_file_name_suffix, + st.session_state.attribute_column, + st.session_state.attribute_column_split, + st.session_state.root_name + ) + st.session_state.data_original = st.session_state.data.copy() + st.session_state.selected_indices = None + st.session_state.project_loaded = True + + elif "view_type" in st.session_state and st.session_state.view_type == "Link Prediction": + st.session_state.data = load_link_data(selected_project_path) + + if st.session_state.data is None and not isinstance(st.session_state.data, dict): + st.warning("⚠️ Not a known Link Prediction project type!") + st.session_state.project_loaded=False + st.stop() + + st.session_state.project_loaded = True + + else: + st.session_state.project_loaded = False + st.warning(f"View type is not implemented yet!") + + # + # AFTER LOADED + # + if st.session_state.project_loaded: + st.session_state.selected_project = selected_project + st.session_state.selected_project_path = selected_project_path + st.success(f"**Project:** `{st.session_state.selected_project}` at `{st.session_state.selected_project_path}`") diff --git a/TELF/applications/Lynx/frontend/pages/tree_view.py b/TELF/applications/Lynx/frontend/pages/tree_view.py new file mode 100644 index 00000000..adcf54ea --- /dev/null +++ b/TELF/applications/Lynx/frontend/pages/tree_view.py @@ -0,0 +1,153 @@ +import streamlit as st +import streamlit.components.v1 as components +from pages.helpers.html_contents import render_tree +from pages.helpers.utils import (prune_tree, filter_documents, + extract_unique_values) + +st.set_page_config(layout="wide") +if ("project_loaded" in st.session_state and st.session_state.project_loaded == False) or "data" not in st.session_state: + st.warning("⚠️ Data has not been loaded yet. Please load the data before accessing this page.") + st.stop() + +if st.session_state.view_type != "Document Analysis": + st.warning("⚠️ Document Analysis view mode is not selected.") + st.stop() + +st.header(st.session_state.selected_project) +with st.sidebar: + st.header("Token Search") + + top_n = st.selectbox("Top n:", ["All"] + list(range(1, 101)), index=0) + keywords_search_type = st.selectbox("Word Search Type:", ["Keywords", "Denovo"], index=0) + if keywords_search_type == "Keywords": + token_search_field = "all_words" + else: + token_search_field = "all_words_denovo" + + # Gather all unique words + words = list( + set( + w + for wlist in st.session_state.documents_data[token_search_field].values() + for w in wlist + ) + ) + words.sort() + + selected_words = st.multiselect("Select words to search:", words) + negative_words = st.multiselect("Select negative words:", words) + token_search_type = st.radio("Search type:", ["and", "or"]) + + # --------------------------------------------------- + # Attribute Search + st.header("Attribute Search") + + search_categories = { + "Countries": { + "options": extract_unique_values("all_countries", st.session_state.documents_data), + "key": "country" + }, + "Authors": { + "options": extract_unique_values("all_authors", st.session_state.documents_data), + "key": "author" + }, + "Affiliations": { + "options": extract_unique_values("all_affiliations", st.session_state.documents_data), + "key": "affiliation" + }, + f"Attribute {st.session_state.attribute_column}": { + "options": extract_unique_values("all_attributes", st.session_state.documents_data), + "key": "attributes" + } + } + + # Build UI for attribute selections + attr_selections = {} + for label, info in search_categories.items(): + with st.expander(f"Search by {label}"): + attr_key = info["key"] + attr_selections[f"selected_{attr_key}"] = st.multiselect( + f"Select {label.lower()}:", info["options"] + ) + attr_selections[f"{attr_key}_search_type"] = st.radio( + "Search type:", ["and", "or"], key=f"{attr_key}_radio" + ) + # --------------------------------------------------- + filter_df_setting = False + + # --------------------------------------------------- + # Display Settings + st.header("Display") + + selected_contents = st.multiselect( + "**Shown Content:**", + options=["Keyword", "Denovo", "Author", "Affiliation", "Country", "Attribute"], + default=["Keyword", "Country"] + ) + top_n_content_shown = st.selectbox("**Top n shown:**", + ["All"] + list(range(1, 101)), index=10) + + with st.expander(f"Coloring"): + selected_visibility = st.radio( + "**Font Color:**", + ["Dark", "Light"], + index=0, + horizontal=True, + ) + contents_coloring = st.radio( + "**Contents Coloring:**", + ["No Color", "tab20", + "turbo", "hsv", + "viridis", "plasma", + "cividis" + ], + index=1, + horizontal=True, + ) + +any_token_selected = bool(selected_words) +any_negative_token_selected = bool(negative_words) +any_attr_selected = any( + attr_selections.get(f"selected_{cfg['key']}", []) + for cfg in search_categories.values() +) + +# If nothing is selected, reset the data +if not any_token_selected and not any_attr_selected and not negative_words: + st.session_state.data = st.session_state.data_original.copy() + st.session_state.selected_indices = None +else: + # Filter documents once based on both token and attribute selections + selected_labels, selected_indices = filter_documents( + data_map=st.session_state.data_map, + top_n=top_n, + selected_words=selected_words, + token_search_type=token_search_type, + attr_selections=attr_selections, + search_categories=search_categories, + keywords_search_type=keywords_search_type, + use_index_map=filter_df_setting, + negative_words=negative_words, + ) + if selected_labels: + st.session_state.data = [prune_tree(st.session_state.data_original[0], selected_labels, st.session_state.root_name)] + st.session_state.selected_indices = selected_indices + else: + # No matches => clear data or handle how you prefer + st.session_state.data = [{"children":[]}] + st.session_state.selected_indices = None + + +#st.write(st.session_state.data_map["0"]) +#st.write(st.session_state.data[0]["children"]) +# display the tree +components.html( + render_tree(tree_data=st.session_state.data[0]["children"], + data_map=st.session_state.data_map, + settings = { + "selected_visibility":selected_visibility, + "selected_contents":selected_contents, + "top_n_content_shown":top_n_content_shown, + "contents_coloring":contents_coloring, + }), + height=800, scrolling=True) \ No newline at end of file diff --git a/TELF/applications/Penguin/crocodile.py b/TELF/applications/Penguin/crocodile.py index dad2666d..3abad583 100644 --- a/TELF/applications/Penguin/crocodile.py +++ b/TELF/applications/Penguin/crocodile.py @@ -18,6 +18,7 @@ import warnings import xmltodict import pandas as pd +import sys pd.options.mode.chained_assignment = None from collections import Counter @@ -325,9 +326,9 @@ def process_s2_json(file_path, output_dir): try: output = json.load(fh) except json.JSONDecodeError as e: - print(f'Failed to decode JSON from {fn!r}: {e}', file=sys.stderr) + print(f'Failed to decode JSON from {file_path!r}: {e}', file=sys.stderr) except Exception as e: - print(f'An error occurred while inserting data from {fn!r}: {e}', file=sys.stderr) + print(f'An error occurred while inserting data from {file_path!r}: {e}', file=sys.stderr) # remove pre-computed embedding so its not taking up space in mongo if 'embedding' in output: @@ -345,7 +346,7 @@ def process_s2_json(file_path, output_dir): if output_dir is None: return output - with open(os.path.join(output_dir, f'{fn}.json'), 'w') as fh: + with open(os.path.join(output_dir, f'{file_path}.json'), 'w') as fh: json.dump(output, fh) @@ -664,7 +665,6 @@ def form_s2_df(data): } for doc in data: - # get the basic information about the document s2id = doc.get('paperId') doi = get_from_dict(doc, ['externalIds', 'DOI']) @@ -675,7 +675,9 @@ def form_s2_df(data): # get author information authors = [] author_ids = [] - for auth in doc.get('authors'): + + for auth in doc.get('authors', []): + auth_id = auth.get('authorId') auth_name = auth.get('name', 'Unknown') if auth_id: @@ -692,7 +694,12 @@ def form_s2_df(data): # get citations citations = None num_citations = None - if doc.get('citations'): + citation_raw = doc.get('citations') + if isinstance(citation_raw, str): + citations = citation_raw.split(';') + num_citations = len(citations) + citations = ';'.join(citations) + elif citation_raw and isinstance(citation_raw, dict): citations = [x.get('paperId', None) for x in doc['citations']] citations = [str(x) for x in citations if x is not None] if not citations: @@ -704,7 +711,13 @@ def form_s2_df(data): # get references references = None num_references = None - if doc.get('references'): + references_raw = doc.get('references') + + if isinstance(references_raw, str): + references = references_raw.split(';') + num_references = len(references) + references = ';'.join(references) + elif references_raw and isinstance(references_raw, dict): references = [x.get('paperId') for x in doc['references']] references = [str(x) for x in references if x is not None] if not references: diff --git a/TELF/applications/Penguin/penguin.py b/TELF/applications/Penguin/penguin.py index be124499..61b5db25 100644 --- a/TELF/applications/Penguin/penguin.py +++ b/TELF/applications/Penguin/penguin.py @@ -6,6 +6,7 @@ from tqdm import tqdm from rbloom import Bloom from joblib import Parallel, delayed +from typing import Sequence from pymongo import MongoClient, ASCENDING from pymongo.errors import ConnectionFailure @@ -329,7 +330,7 @@ def _add_single_s2_file(self, file_path, overwrite=True): if file_path.suffix == '.json': data = process_s2_json(file_path, output_dir=None) else: - raise ValueError(f'S2 `file_path` has an unsupported file extension: {source!r}') + raise ValueError(f'S2 `file_path` has an unsupported file extension: {file_path!r}') collection = self.db[self.S2_COL] id_attr = self.s2_attributes['id'] @@ -375,7 +376,7 @@ def _add_single_scopus_file(self, file_path, overwrite=True): elif file_path.suffix == '.json': data = process_scopus_json(file_path, output_dir=None) else: - raise ValueError(f'Scopus `file_path` has an unsupported file extension: {source!r}') + raise ValueError(f'Scopus `file_path` has an unsupported file extension: {file_path!r}') collection = self.db[self.SCOPUS_COL] id_attr = self.scopus_attributes['id'] @@ -749,7 +750,7 @@ def resolve_document_id(self, pid): """ prefix, _, actual_id = pid.partition(':') if not actual_id: - raise ValueError(f'Encountered unknown `id`: {doc_id}. Prepend id with eid: or s2id: to specify id type') + raise ValueError(f'Encountered unknown `id`: {pid}. Prepend id with eid: or s2id: to specify id type') document = None collection = None @@ -895,6 +896,34 @@ def find_by_tag(self, tag, as_pandas=True): out['s2'] = self.db[self.S2_COL].find({"tags": tag}) return out + + def distinct_field( + self, + source: str, + field: str, + filter_spec: dict | None = None + ) -> set: + """ + Return the set of unique values for `field` in `source` matching `filter_spec`. + If filter_spec is None, returns all distinct values of that field. + """ + coll = self.db[source] + return set(coll.distinct(field, filter_spec or {})) + + def distinct_dois( + self, + source: str = S2_COL, + dois: Sequence[str] | None = None + ) -> set: + """ + Return the subset of `dois` that are already in the database (or all DOIs if `dois` is None). + """ + doi_field = ( + self.s2_attributes['doi'] if source == self.S2_COL + else self.scopus_attributes['doi'] + ) + spec = {doi_field: {'$in': [d.lower() for d in dois]}} if dois else None + return self.distinct_field(source, doi_field, spec) # 5. iPenguin Hook diff --git a/TELF/helpers/data_structures.py b/TELF/helpers/data_structures.py index 5abc3d66..8025db8e 100644 --- a/TELF/helpers/data_structures.py +++ b/TELF/helpers/data_structures.py @@ -277,8 +277,11 @@ def process_countries(row): if isinstance(row, str): row = ast.literal_eval(row) for k, v in row.items(): - countries.append(v.get('country', 'Unknown')) + raw = v.get('country', None) + countries.append(raw if raw is not None else 'Unknown') + # countries.append(v.get('country', 'Unknown')) countries = list(set(countries)) + return ';'.join(countries) def process_affiliations(row): @@ -291,7 +294,9 @@ def process_affiliations(row): row = ast.literal_eval(row) for k, v in row.items(): affil_ids.append(k) - affil_names.append(v.get('name', 'Unknown')) + # affil_names.append(v.get('name', 'Unknown')) + raw = v.get('name', None) + affil_names.append(raw if raw is not None else 'Unknown') return ';'.join(affil_ids), ';'.join(affil_names) def sum_dicts(dict_list, n): diff --git a/TELF/helpers/embeddings.py b/TELF/helpers/embeddings.py index f95dba3a..b875522d 100644 --- a/TELF/helpers/embeddings.py +++ b/TELF/helpers/embeddings.py @@ -1,25 +1,12 @@ -from transformers import AutoTokenizer, AutoModel + import torch -from openai import OpenAI -import string + import warnings from tqdm import tqdm import numpy as np from sklearn.metrics import pairwise_distances -def get_model_name(model_name): - assert model_name in ('SCINCL', 'SPECTER'), f'Unknown model name "{model_name}"!' - if model_name == 'SCINCL': - return 'malteos/scincl' - if model_name == 'SPECTER': - return 'allenai/specter2_base' - -def initialize_model_and_device(embedding_model: str, use_gpu: bool = False): - device = 'cuda:0' if use_gpu and torch.cuda.is_available() else 'cpu' - tokenizer = AutoTokenizer.from_pretrained(get_model_name(embedding_model)) - model = AutoModel.from_pretrained(get_model_name(embedding_model)).to(device) - return tokenizer, model, device - +from .llm_models import get_transformer_llm def compute_doc_embedding(text, *, model=None, tokenizer=None, model_name='SCINCL', device='cpu', max_length=512): use_gpu = True @@ -27,34 +14,18 @@ def compute_doc_embedding(text, *, model=None, tokenizer=None, model_name='SCINC use_gpu = False if model is None and tokenizer is None: - tokenizer, model, device = initialize_model_and_device(model_name, use_gpu) + tokenizer, model, device = get_transformer_llm(model_name, use_gpu) inputs = tokenizer(text, padding=True, truncation=True, return_tensors="pt", max_length=max_length).to(device) result = model(**inputs) - if not use_gpu: - return torch.mean(result.last_hidden_state[0], dim=0).detach().numpy() - else: - return torch.mean(result.last_hidden_state[0], dim=0).cpu().detach().numpy() - - - -def produce_label(openai_api_key: str, words: str): - prompt = ( - f"Use the following words to establish a theme or a topic: {', '.join(words)!r}. " - "The output should be a label that has at most 3 tokens and is characterized by the provided words. " - "The output should not be in quotes." - ) - client = OpenAI(api_key=openai_api_key) - response = client.completions.create( - model="gpt-3.5-turbo-instruct", - prompt=prompt, - max_tokens=2048 - ) - label = response.choices[0].text.strip() - return label.strip().translate(str.maketrans('', '', string.punctuation + '\n')) + return result.last_hidden_state[0].mean(0).detach().cpu().numpy() + # if not use_gpu: + # return torch.mean(result.last_hidden_state[0], dim=0).detach().numpy() + # else: + # return torch.mean(result.last_hidden_state[0], dim=0).cpu().detach().numpy() def compute_embeddings(df, *, model_name='SCINCL', cols=['title', 'abstract'], sep_token='[SEP]', as_np=False, use_gpu=True): - tokenizer, model, device = initialize_model_and_device(model_name, use_gpu) + tokenizer, model, device = get_transformer_llm(model_name, use_gpu) if use_gpu and device == 'cpu': warnings.warn(f'Tried to use GPU, but GPU is not available. Using {device}.') diff --git a/TELF/helpers/figures.py b/TELF/helpers/figures.py index 287ba931..671c42f9 100644 --- a/TELF/helpers/figures.py +++ b/TELF/helpers/figures.py @@ -16,6 +16,7 @@ from matplotlib.colors import to_hex from scipy.spatial import ConvexHull import logging +import math log = logging.getLogger(__name__) from .file_system import check_path from .data_structures import sum_dicts @@ -112,50 +113,37 @@ def plot_authors_graph(df, id_col='s2_author_ids', name_col='s2_authors', title= ) return fig -def create_wordcloud_from_df(df, col, n=30, save_path=None, figsize=(600,600)): +def create_wordcloud_from_df(df, col, n=30, save_path=None, figsize=(6, 6)): """ - Generate and display a word cloud from a specified DataFrame column. - - Parameters: - ----------- - df: pd.DataFrame - The DataFrame containing the text data. - col: str - The column name in the DataFrame with the text data. - n: int, optional - The number of top words to consider for the word cloud. Default is 30. - save_path: str, optional - Path to save the word cloud image. If not provided, the image is displayed but not saved. - figsize: (int, int), optional - A tuple specifying the size of the displayed image in inches. Default is (600,600). - - Returns: - -------- - None: - Displays the word cloud using matplotlib. If save_path is provided, the word cloud is saved to the - specified path and not displayed. + Generate a word cloud from `df[col]` and either display it + or save it to `save_path`. """ - # calculate the frequencies - tokens = [x.split() for x in df[col].to_list() if not pd.isna(x)] - top_words = [dict(Counter(x)) for x in tokens] - top_words = sum_dicts(top_words, n=n) - - # generate the word cloud image - figsize_x, figsize_y = figsize - wc = WordCloud(background_color="white", width=figsize_x*100, height=figsize_y*100) - wc.generate_from_frequencies(top_words) - - # display the word cloud using matplotlib - plt.figure(figsize=figsize) - plt.imshow(wc, interpolation="bilinear") - plt.axis("off") - - if save_path is not None: - plt.savefig(save_path, dpi=300) - plt.close() + # 1) Build frequencies --------------------------------------------------- + tokens = [s.split() for s in df[col].dropna()] + freq = Counter(word for row in tokens for word in row).most_common(n) + freq = dict(freq) + + # 2) Generate cloud ------------------------------------------------------ + wc = WordCloud(background_color="white", + width=int(figsize[0]*200), + height=int(figsize[1]*200)) + wc.generate_from_frequencies(freq) + + # 3) Plot on a *local* figure/axes (thread-safe) ------------------------- + fig, ax = plt.subplots(figsize=figsize, dpi=200) + ax.imshow(wc, interpolation="bilinear") + ax.axis("off") + fig.tight_layout() + + # 4) Save or show -------------------------------------------------------- + if save_path: + Path(save_path).parent.mkdir(parents=True, exist_ok=True) + fig.savefig(save_path, bbox_inches="tight", pad_inches=0.1) else: plt.show() + plt.close(fig) + def _cmap_map(function, cmap): """ Applies function (which should operate on vectors of shape 3: [r, g, b]), on colormap cmap. diff --git a/TELF/helpers/file_system.py b/TELF/helpers/file_system.py index 92747bee..fee8a567 100644 --- a/TELF/helpers/file_system.py +++ b/TELF/helpers/file_system.py @@ -5,6 +5,9 @@ import csv import shutil import fnmatch +import shutil +from pathlib import Path +from typing import Union def load_file_as_dict(fn): """ @@ -266,4 +269,45 @@ def copy_file(source_dir, dest_dir, file_pattern): try: shutil.copy(os.path.join(source_dir, matching_files[0]), dest_dir) except IOError as e: - raise IOError(f"Failed to copy file: {e}") \ No newline at end of file + raise IOError(f"Failed to copy file: {e}") + + + +def copy_all_files(src_dir: str, dst_dir: str, preserve_metadata: bool = True) -> None: + """ + Recursively copy everything under src_dir into dst_dir. + If a file fails to copy due to permissions (or any OSError), log+skip it. + """ + src_dir = os.fspath(src_dir) + dst_dir = os.fspath(dst_dir) + skipped = [] + + for root, dirs, files in os.walk(src_dir): + rel = os.path.relpath(root, src_dir) + target_root = os.path.join(dst_dir, rel) + os.makedirs(target_root, exist_ok=True) + + # create subdirs + for d in dirs: + os.makedirs(os.path.join(target_root, d), exist_ok=True) + + for name in files: + src_path = os.path.join(root, name) + dst_path = os.path.join(target_root, name) + + try: + if preserve_metadata: + shutil.copy2(src_path, dst_path) + else: + shutil.copy(src_path, dst_path) + except OSError as e: + skipped.append(src_path) + # we skip this file and continue + continue + + if skipped: + warnings.warn( + f"copy_all_files: skipped {len(skipped)} files due to permissions or IO errors. " + f"First few: {skipped[:5]!r}" + ) + diff --git a/TELF/helpers/filters.py b/TELF/helpers/filters.py index b194c913..860c9464 100644 --- a/TELF/helpers/filters.py +++ b/TELF/helpers/filters.py @@ -1,5 +1,8 @@ +from typing import Any, Sequence import pandas as pd import ast +import re + def get_papers(df, authors, col): """ @@ -67,4 +70,115 @@ def filter_author_map(df, *, country=None, affiliation=None): if affiliation is not None and affiliation not in affiliation_ids: ids.remove(index) - return df.iloc[list(ids)] \ No newline at end of file + return df.iloc[list(ids)] + + +def find_subdf(df: pd.DataFrame, column: str, ids: Sequence[str]) -> pd.DataFrame: + """ + Filter a DataFrame for rows where a column contains any of the specified IDs + as semicolon‐separated tokens. + + Parameters + ---------- + df : pandas.DataFrame + Input DataFrame. + column : str + Name of the column in `df` containing semicolon‐separated identifiers. + ids : sequence of str + List of identifier strings to match as whole tokens. + + Returns + ------- + pandas.DataFrame + Subset of `df` containing only rows where `column` contains any of the + specified `ids` (delimited by semicolons or string boundaries). + """ + escaped_ids = [re.escape(id_) for id_ in ids] + pattern = r'(?:^|;\s*)(' + '|'.join(escaped_ids) + r')(?:\s*;|$)' + mask = df[column].str.contains(pattern, regex=True) + return df[mask] + + +def remove_unknown_entries(cell: Any) -> Any: + """ + Remove dictionary‐like 'Unknown' entries from a string, cleaning up commas. + + This function removes any substring matching + `[, ]?'Unknown': [ ... ]` and then tidies up leftover commas. + + Parameters + ---------- + cell : any + If not a string, returned unchanged. If a string, cleaned. + + Returns + ------- + any + The cleaned string with 'Unknown' entries removed, or the original + non‐string `cell`. + """ + if not isinstance(cell, str): + return cell + + s = cell + # Remove "'Unknown': [...]" entries (with optional leading comma) + s = re.sub(r",?\s*'Unknown'\s*:\s*\[[^\]]*\]", "", s) + # Clean up trailing commas before braces or ends + s = re.sub(r",\s*}", "}", s) + s = re.sub(r"^,\s*", "", s) + s = re.sub(r",\s*$", "", s) + return s.strip() + + + + + +def normalize_affiliation_cell(cell): + """ + Turn whatever is in `cell` into a dict of affiliation‐info dicts. + - If it’s already a dict, return as-is. + - If it’s a list, enumerate it into a dict. + - If it’s a string, optionally clean out Unknowns, then ast.literal_eval it. + - Otherwise return {}. + """ + if isinstance(cell, dict): + return cell + + if isinstance(cell, list): + # turn [info0, info1, …] into {0:info0, 1:info1, …} + return {i: entry for i, entry in enumerate(cell)} + + if isinstance(cell, str): + # 1) strip out 'Unknown':[…] patterns + s = remove_unknown_entries(cell) + # 2) now parse it safely back into a Python object + try: + parsed = ast.literal_eval(s) + except Exception: + return {} + # 3) if the result is a list or dict, normalize it: + return normalize_affiliation_cell(parsed) + + # anything else → no affiliations + return {} + +def clean_affiliations(df: pd.DataFrame) -> pd.DataFrame: + """ + Clean the 'affiliations' column by removing unknown entries in each cell. + + Applies `remove_unknown_entries` to every entry of the 'affiliations' column. + + Parameters + ---------- + df : pandas.DataFrame + DataFrame with an 'affiliations' column. + + Returns + ------- + pandas.DataFrame + The same DataFrame with 'affiliations' cleaned in‐place. + """ + df['affiliations'] = df['affiliations'].apply(remove_unknown_entries) + df['affiliations'] = df['affiliations'].apply(normalize_affiliation_cell) + + return df diff --git a/TELF/helpers/frames.py b/TELF/helpers/frames.py index 0fe4b894..44b7bf91 100644 --- a/TELF/helpers/frames.py +++ b/TELF/helpers/frames.py @@ -3,6 +3,20 @@ import rapidfuzz import uuid import warnings +import re +from typing import Sequence +from tqdm import tqdm +import ast +from .data_structures import process_countries, process_affiliations +from sklearn.feature_extraction.text import TfidfVectorizer +from typing import List + +ALPHA = 1e-12 +def apply_alpha(x): + if isinstance(x, (int, float)): + return 0 if abs(x) < ALPHA else x + else: + return x def create_coauthor_affil_df(auth_affil_map, name_map): data = { @@ -241,8 +255,8 @@ def merge_frames(df1, df2, col, common_cols, key=None, remove_duplicates=False, pd.DataFrame: Merged DataFrame """ - df1[col] = df1[col].str.lower() - df2[col] = df2[col].str.lower() + df1[col] = df1[col].astype(str).str.lower() + df2[col] = df2[col].astype(str).str.lower() merged_df = df1.merge(df2, on=col, how='outer', suffixes=('_df1', '_df2')) @@ -262,4 +276,199 @@ def merge_frames(df1, df2, col, common_cols, key=None, remove_duplicates=False, if order is not None: # reset order (optional) merged_df = merged_df[order].reset_index(drop=True) - return merged_df \ No newline at end of file + return merged_df + + +def add_num_known_col(slic_df): + slic_df['num_papers'] = [0] * len(slic_df) + for index, row in tqdm(slic_df.iterrows(), total=len(slic_df)): + affiliations = row.scopus_affiliations + if pd.isna(affiliations): + continue + if isinstance(affiliations, str): + affiliations = ast.literal_eval(affiliations) + + num_papers = len(set.union(*[set(x['papers']) for x in affiliations.values()])) + slic_df.at[index, 'num_papers'] = num_papers + return slic_df + +def prep_affiliations(df): + df[['affiliation_ids', 'affiliation_names']] = df['slic_affiliations'].apply(lambda row: pd.Series(process_affiliations(row))) + # df['countries'] = df['slic_affiliations'].apply(process_countries) + + df['countries'] = df['slic_affiliations'].map(lambda row: pd.Series(process_countries(row))[0]) + df = df.dropna(subset=['affiliation_ids', 'affiliation_names', 'countries']).reset_index(drop=True) + return df + +def term_frequency(df, term, col): + """ + Count total occurrences of `term` in column `col` of DataFrame `df`, + treating `term` as a literal substring (no regex specials). + """ + pattern = re.escape(term.lower()) + return df[col] \ + .str.lower() \ + .str.count(pattern) \ + .sum() + +def document_frequency(df, term, col): + """ + Count how many rows in column `col` contain `term` at least once, + treating `term` as a literal substring. + """ + pattern = re.escape(term.lower()) + return df[col] \ + .str.lower() \ + .str.contains(pattern, na=False) \ + .sum() + +def calculate_term_representations(df, terms, col): + """ + Calculate term frequency, document frequency, and TF-IDF scores + for a list of terms in a pandas DataFrame. + + Parameters: + ----------- + df: pd.DataFrame + Pandas DataFrame with a column col containing the text data. + col: str + The column in which to search for terms + terms: list + List of terms to calculate TF-IDF scores, term frequency, and document frequency for. + + Returns: + -------- + pd.DataFrame + A new DataFrame with columns 'Term', 'Term Frequency', 'Document Frequency', 'TF-IDF Score'. + """ + vectorizer = TfidfVectorizer(vocabulary=terms, dtype=np.float32) + tfidf_matrix = vectorizer.fit_transform(df[col].dropna()) + + # get feature names (the terms vocabulary) from vectorizer + feature_names = list(vectorizer.get_feature_names_out()) + + # calculate average TF-IDF score for each term + avg_tfidf_scores = tfidf_matrix.mean(axis=0).tolist()[0] + + # prepare results DataFrame + results_df = { + 'Term': [], + 'Term Frequency': [], + 'Document Frequency': [], + 'TF-IDF Score': [] + } + + # calculate TF, DF for each term + for term in terms: + term_index = feature_names.index(term) + tf = term_frequency(df, term, col) + df_count = document_frequency(df, term, col) + tfidf_score = avg_tfidf_scores[term_index] + + results_df['Term'].append(term) + results_df['Term Frequency'].append(tf) + results_df['Document Frequency'].append(df_count) + results_df['TF-IDF Score'].append(tfidf_score) + + return pd.DataFrame.from_dict(results_df) + + +def clean_duplicates(df): + duplicates = df.duplicated(subset=['doi', 'title', 'abstract'], keep=False) + conflicts = df.duplicated(subset=['doi'], keep=False) & ~duplicates + df = df[~df['doi'].isin(df[conflicts]['doi'])] + df = df.drop_duplicates(subset=['doi', 'title', 'abstract'], keep='first') + return df + + +def merge_scopus_s2(df_scopus: pd.DataFrame, + s2_df: pd.DataFrame, + df_order=[ + 'eid', 's2id', 'doi', 'title', 'abstract', 'year', 'authors', 'author_ids', + 'affiliations', 'funding', 'PACs', 'publication_name', 'subject_areas', + 's2_authors', 's2_author_ids', 'citations', 'references', + 'num_citations', 'num_references' + ] + ) -> pd.DataFrame: + # Ensure 'doi' exists in both DataFrames + if 'doi' not in df_scopus.columns: + df_scopus['doi'] = np.nan + if 'doi' not in s2_df.columns: + s2_df['doi'] = np.nan + + # Ensure all columns exist in both DataFrames before merging + for col in df_order: + if col not in df_scopus.columns: + df_scopus[col] = np.nan + if col not in s2_df.columns: + s2_df[col] = np.nan + + # Perform merging, ensuring 'doi' is used as the key, not in common_cols + merged_df = merge_frames( + df1=df_scopus, + df2=s2_df, + col='doi', + common_cols=[col for col in df_order if col != 'doi'], # Exclude 'doi' from common_cols + remove_duplicates=True, + remove_nan=True, + order=None + ) + # Ensure merged_df has all required columns + for col in df_order: + if col not in merged_df.columns: + merged_df[col] = np.nan + # Reorder columns and reset index + merged_df = merged_df[df_order].reset_index(drop=True) + merged_df.info() + return merged_df + + +def drop_columns_if_exist(df: pd.DataFrame, cols: List[str], inplace: bool = False) -> pd.DataFrame: + to_drop = [c for c in cols if c in df.columns] + if not to_drop: + return None if inplace else df.copy() + + if inplace: + df.drop(columns=to_drop, inplace=True) + return None + else: + return df.drop(columns=to_drop) + + +def calculate_term_attribution( + df: pd.DataFrame, + terms: Sequence[str], + col: str +) -> pd.DataFrame: + """ + Add a `term_attribution` column to `df` consisting of the + terms (from the provided list) that appear in each row’s `col`. + + Parameters: + ----------- + df : pd.DataFrame + DataFrame with a text column named `col`. + terms : Sequence[str] + List of terms to search for in each text. + col : str + Name of the column in which to search for terms. + + Returns: + -------- + pd.DataFrame + The same DataFrame with an extra column `term_attribution` + which is a string of matching terms joined by ', '. + """ + # ensure all terms are lowercased once + terms_lc = [t.lower() for t in terms] + + def _find_terms(text: str) -> str: + if not isinstance(text, str) or not text: + return "" + text_lc = text.lower() + matched = [terms[i] for i, t in enumerate(terms_lc) if t in text_lc] + return ", ".join(matched) + + df = df.copy() + df["term_attribution"] = df[col].apply(_find_terms) + return df diff --git a/TELF/helpers/llm.py b/TELF/helpers/llm.py deleted file mode 100644 index 19a99eee..00000000 --- a/TELF/helpers/llm.py +++ /dev/null @@ -1,68 +0,0 @@ -import json -import re -import logging -from langchain_ollama import OllamaLLM -import subprocess - -log = logging.getLogger(__name__) -from typing import Iterable - -def build_json_vote_prompt(candidate: str, contexts: Iterable[str]) -> str: - """ - Build a JSON-only prompt from example contexts and a candidate string. - """ - ctx_block = "\n----\n".join(contexts) - return ( - "You are an expert researcher. Output ONLY valid JSON.\n" - f"Target context examples:\n{ctx_block}\n\n" - f"Candidate abstract:\n{candidate}\n" - "Given the context, is the candidate about any of the concepts? " - 'Respond {"answer":"yes|no","reason":"..."}' - ) - - -def get_ollama_llm(model: str, base_url: str, temperature: float) -> OllamaLLM: - """ - Create and return a configured OllamaLLM instance. - If `model` isn’t yet pulled locally, this will shell out to: - ollama pull - """ - try: - available = subprocess.check_output( - ["ollama", "list"], text=True, stderr=subprocess.DEVNULL - ) - except subprocess.CalledProcessError as e: - raise RuntimeError(f"Could not list Ollama models: {e}") - - if model not in available: - try: - print(f"Model '{model}' not found locally – pulling…") - subprocess.run( - ["ollama", "pull", model], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - except subprocess.CalledProcessError as e: - raise RuntimeError(f"Failed to pull Ollama model '{model}': {e.stderr.strip()}") - - return OllamaLLM(model=model, base_url=base_url, temperature=temperature) - - - -def vote_once(llm: OllamaLLM, prompt: str) -> tuple[bool, str]: - """ - Invoke the given OllamaLLM instance once, strip markdown fences, - parse its JSON response, and return (yes_flag, reason). - """ - raw = llm.invoke(prompt) - txt = re.sub(r"```(?:json)?", "", raw).strip() - try: - obj = json.loads(txt) - yes = obj.get("answer", "").lower() == "yes" - reason = str(obj.get("reason", "")).strip() - return yes, reason - except Exception: - log.warning("Bad JSON from LLM: %s", raw) - return False, "" \ No newline at end of file diff --git a/TELF/helpers/llm_models.py b/TELF/helpers/llm_models.py new file mode 100644 index 00000000..5b7cd52b --- /dev/null +++ b/TELF/helpers/llm_models.py @@ -0,0 +1,61 @@ +import subprocess +from openai import OpenAI +import subprocess +from transformers import AutoTokenizer, AutoModel +import torch +from langchain_ollama import OllamaLLM + +def get_ollama_llm(model: str, base_url: str, temperature: float) -> OllamaLLM: + """ + Create and return a configured OllamaLLM instance. + If `model` isn’t yet pulled locally, this will shell out to: + ollama pull + """ + try: + available = subprocess.check_output( + ["ollama", "list"], text=True, stderr=subprocess.DEVNULL + ) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Could not list Ollama models: {e}") + + if model not in available: + try: + print(f"Model '{model}' not found locally – pulling…") + subprocess.run( + ["ollama", "pull", model], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to pull Ollama model '{model}': {e.stderr.strip()}") + + return OllamaLLM(model=model, base_url=base_url, temperature=temperature) + +def get_openai_llm(openai_api_key:str): + client = OpenAI(api_key=openai_api_key) + return client + +def get_model_name(model_name): + assert model_name in ('SCINCL', 'SPECTER'), f'Unknown model name "{model_name}"!' + if model_name == 'SCINCL': + return 'malteos/scincl' + if model_name == 'SPECTER': + return 'allenai/specter2_base' + return model_name + +def get_transformer_llm(embedding_model: str, device=False): + if isinstance(device, int) and torch.cuda.is_available(): + device = f'cuda:{device}' + # this is hackish, needs a fix, but im not doing it right now + if 'false' in device.lower(): + device = 'cpu' + elif 'true' in device.lower(): + device = 'cuda:0' + else: + device = 'cuda:0' if device and torch.cuda.is_available() else 'cpu' + print(f'Using device: {device}') + tokenizer = AutoTokenizer.from_pretrained(get_model_name(embedding_model)) + model = AutoModel.from_pretrained(get_model_name(embedding_model)).to(device) + return tokenizer, model, device \ No newline at end of file diff --git a/TELF/helpers/llm_operations.py b/TELF/helpers/llm_operations.py new file mode 100644 index 00000000..56aaa5f3 --- /dev/null +++ b/TELF/helpers/llm_operations.py @@ -0,0 +1,35 @@ +import json +import re +import logging +from langchain_ollama import OllamaLLM +import string + +from .prompts import produce_label_prompt +log = logging.getLogger(__name__) + +def vote_once(llm: OllamaLLM, prompt: str) -> tuple[bool, str]: + """ + Invoke the given OllamaLLM instance once, strip markdown fences, + parse its JSON response, and return (yes_flag, reason). + """ + raw = llm.invoke(prompt) + txt = re.sub(r"```(?:json)?", "", raw).strip() + try: + obj = json.loads(txt) + yes = obj.get("answer", "").lower() == "yes" + reason = str(obj.get("reason", "")).strip() + return yes, reason + except Exception: + log.warning("Bad JSON from LLM: %s", raw) + return False, "" + +def produce_label(words: str, client=None, model="gpt-3.5-turbo-instruct"): + prompt = produce_label_prompt(words) + + response = client.completions.create( + model=model, + prompt=prompt, + max_tokens=2048 + ) + label = response.choices[0].text.strip() + return label.strip().translate(str.maketrans('', '', string.punctuation + '\n')) \ No newline at end of file diff --git a/TELF/helpers/prompts.py b/TELF/helpers/prompts.py new file mode 100644 index 00000000..8b742724 --- /dev/null +++ b/TELF/helpers/prompts.py @@ -0,0 +1,22 @@ +from typing import Iterable + +def build_json_vote_prompt(candidate: str, contexts: Iterable[str]) -> str: + """ + Build a JSON-only prompt from example contexts and a candidate string. + """ + ctx_block = "\n----\n".join(contexts) + return ( + "You are an expert researcher. Output ONLY valid JSON.\n" + f"Target context examples:\n{ctx_block}\n\n" + f"Candidate abstract:\n{candidate}\n" + "Given the context, is the candidate about any of the concepts? " + 'Respond {"answer":"yes|no","reason":"..."}' + ) + +def produce_label_prompt(words:Iterable[str]): + prompt = ( + f"Use the following words to establish a theme or a topic: {', '.join(words)!r}. " + "The output should be a label that has at most 3 tokens and is characterized by the provided words. " + "The output should not be in quotes." + ) + return prompt diff --git a/TELF/helpers/terms.py b/TELF/helpers/terms.py index d7e8a349..b0bfcc36 100644 --- a/TELF/helpers/terms.py +++ b/TELF/helpers/terms.py @@ -1,3 +1,4 @@ +from __future__ import annotations import warnings import pathlib @@ -5,6 +6,10 @@ from ..pre_processing.Vulture.default_stop_words import STOP_WORDS from ..pre_processing.Vulture.default_stop_phrases import STOP_PHRASES +from ..pre_processing.Vulture.modules import LemmatizeCleaner +from typing import Any, Callable, Dict, List, Tuple, Optional,Sequence + + def load_stop_terms(terms, words=True): """ Load the stop words or stop phrases depending on the type of `terms` @@ -245,3 +250,77 @@ def unite_terms(*args): result.append({key: values}) result.extend(unique_strings) return result + + + +def resolve_substitution_conflicts( + sub_map: Dict[str, Any], + *, + resolver: Callable[[List[Tuple[str, Any]]], Tuple[str, Any]] | None = None, + warn: bool = True, +) -> tuple[Dict[str, Any], set[str]]: + """ + Collapse keys that lemmatize to the same token but map to different + targets. + + Returns + ------- + cleaned : Dict[str, Any] + Conflict-free forward map (orig -> replacement). + dropped : set[str] + Every orig key that was removed during conflict resolution. + """ + # ------------------------------------------------------------------ # + # 1) pick a lemmatizer # + # ------------------------------------------------------------------ # + try: + lemmatizer = LemmatizeCleaner(library="spacy") + except TypeError: + lemmatizer = LemmatizeCleaner(library="nltk") # safe fallback + + # ------------------------------------------------------------------ # + # 2) default conflict policy: keep the *shortest* original string # + # ------------------------------------------------------------------ # + if resolver is None: + resolver = lambda items: min(items, key=lambda kv: len(kv[0])) + + # ------------------------------------------------------------------ # + # 3) bucket originals by their normalized (lemmatized) form # + # ------------------------------------------------------------------ # + buckets: Dict[str, List[Tuple[str, Any]]] = {} + for orig, val in sub_map.items(): + try: + _, cleaned = lemmatizer((None, orig)) + norm = cleaned.lower() + except Exception as e: + if warn: + print(f"[resolve_subs] warn: could not lemmatize “{orig}” ({e})") + norm = orig.lower() + + buckets.setdefault(norm, []).append((orig, val)) + + # ------------------------------------------------------------------ # + # 4) resolve each bucket # + # ------------------------------------------------------------------ # + cleaned: Dict[str, Any] = {} + dropped: set[str] = set() + + for norm, items in buckets.items(): + vals = {v for _, v in items} + + if len(vals) == 1: + # unanimous → keep *all* originals + cleaned.update(items) + else: + # conflict → call resolver, drop the rest + keep_orig, keep_val = resolver(items) + cleaned[keep_orig] = keep_val + + toss = [o for o, _ in items if o != keep_orig] + dropped.update(toss) + + if warn: + print(f"[resolve_subs] conflict on {norm!r}: keeping " + f"'{keep_orig}', dropping {toss}") + + return cleaned, dropped diff --git a/TELF/pipeline/__init__.py b/TELF/pipeline/__init__.py new file mode 100644 index 00000000..fb2acca9 --- /dev/null +++ b/TELF/pipeline/__init__.py @@ -0,0 +1,65 @@ +import sys +sys.path += ["blocks"] + +from .block_manager import BlockManager +from .dir_loop_block import DirectoryLoopBlock +from .blocks.base_block import AnimalBlock +from .blocks.data_bundle import ( + DataBundle, + SAVE_DIR_BUNDLE_KEY, + DIR_LIST_BUNDLE_KEY, + RESULTS_DEFAULT, + SOURCE_DIR_BUNDLE_KEY +) + +from .blocks.vulture_clean_block import VultureCleanBlock +from .blocks.beaver_vocab_block import BeaverVocabBlock +from .blocks.orca_block import OrcaBlock +from .blocks.function_block import FunctionBlock +from .blocks.cheetah_filter_block import CheetahFilterBlock +from .blocks.hnmfk_block import HNMFkBlock +from .blocks.nmfk_block import NMFkBlock +from .blocks.wolf_block import WolfBlock +from .blocks.scopus_block import ScopusBlock +from .blocks.s2_block import S2Block +from .blocks.auto_bunny_block import AutoBunnyBlock +from .blocks.beaver_doc_word_block import BeaverDocWordBlock +from .blocks.squirrel_block import SquirrelBlock +from .blocks.beaver_codependency_matrix_block import CodependencyMatrixBlock +from .blocks.dataframe_load_block import LoadDfBlock +from .blocks.field_filter_block import FilterByCountBlock +from .blocks.artic_fox_block import ArticFoxBlock +from .blocks.fox_block import FoxBlock +from .blocks.lmf_block import LMFBlock +from .blocks.split_block import SPLITBlock +from .blocks.split_transfer_block import SPLITTransferBlock +from .blocks.wordcloud_block import WordCloudBlock +from .blocks.load_terms_block import LoadTermsBlock +from .blocks.term_attribution_block import TermAttributionBlock +from .blocks.term_search_block import TermSearchBlock +from .blocks.scores_block import ComputeScoresBlock +from .blocks.sample_matrix_mask_block import SampleMatrixMaskBlock +from .blocks.clean_duplicates_block import CleanDuplicatesBlock +from .blocks.merge_scopus_s2_block import MergeScopusS2Block +from .blocks.clean_affiliations_block import CleanAffiliationsBlock +from .blocks.sbatch_block import SBatchBlock +from .blocks.beaver_cooccurrence_block import BeaverCooccurrenceBlock +from .blocks.beaver_something_words_block import BeaverSomethingWordsBlock +from .blocks.beaver_something_words_time_block import BeaverSomethingWordsTimeBlock +from .blocks.beaver_cocitation_tensor_block import BeaverCocitationTensorBlock +from .blocks.beaver_coauthor_tensor_block import BeaverCoauthorTensorBlock +from .blocks.beaver_citation_tensor_block import BeaverCitationTensorBlock +from .blocks.beaver_participation_tensor_block import BeaverParticipationTensorBlock +from .blocks.rescalk_block import RESCALkBlock +from .blocks.combine_dir_loop_df_block import ConcatenateDFBlock +from .blocks.pipeline_summary_block import PipelineSummaryBlock +from .blocks.author_lines_block import ExpandAuthorAffiliationBlock + +from .loop_block import RepeatLoopBlock +from .blocks.uniqueness_block import UniquenessDFBlock +from .blocks.core_register_block import CoreRegister +from .blocks.group_summary_block import GroupSummaryBlock +from .blocks.post_process_cluster_analysis_block import ClusteringAnalyzerBlock +from .blocks.post_process_label_analysis_block import LabelAnalyzerBlock +from .blocks.peacock_stats_block import PeacockStatsBlock + \ No newline at end of file diff --git a/TELF/pipeline/block_manager.py b/TELF/pipeline/block_manager.py new file mode 100644 index 00000000..032a304b --- /dev/null +++ b/TELF/pipeline/block_manager.py @@ -0,0 +1,404 @@ +from __future__ import annotations + +from pathlib import Path +import sys, io, contextlib, datetime as _dt, traceback +from typing import List, Tuple, Sequence, Any, Dict, Set + +from IPython.display import display, HTML + +from .blocks.base_block import AnimalBlock +from .blocks.data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY, SOURCE_DIR_BUNDLE_KEY + +import jsonpickle +import re +import time + + +class BlockManager: + """ + Orchestrates a list of `AnimalBlock` instances, moving a single + `DataBundle` through the pipeline. + + The manager is **namespace-aware**: + + - Each block owns a unique `tag` (defaults to the class name; + duplicates automatically receive a numeric suffix). + - A block that writes `provides = ("df",)` actually puts its data + under the key ``".df"`` *and* updates the bundle’s + “latest” pointer for ``"df"``. + - A block may depend on either the generic key ( `"df"` ) or any + fully-qualified key ( e.g. `"Clean.df"` ). + """ + + + def __init__( + self, + blocks: List[AnimalBlock], + databundle: DataBundle | None = None, + *, + verbose: bool = True, + progress: bool = True, + capture_output: str | None = "file", + force_checkpoint: bool | None = None, + ): + self.verbose = bool(verbose) + self.progress = bool(progress) + self.capture_output = capture_output # None | "memory" | "file" + self.force_checkpoint = force_checkpoint # None: default; True: force load; False: skip load + self.block_logs: Dict[str, str] = {} + + self.blocks: List[AnimalBlock] = blocks + self.bundle: DataBundle = databundle or DataBundle() + + self._assign_unique_tags() + self._ensure_result_path() + self.check_io_consistency() + if self.verbose: + self.describe_io() + + def __call__(self) -> DataBundle: + total = len(self.blocks) + log_dir: Path | None = None + progress_fp = None + if self.capture_output == "file": + log_dir = Path(self.bundle[SAVE_DIR_BUNDLE_KEY]) / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + progress_fp = (log_dir / "progress.log").open("a", encoding="utf-8") + + table_lines = self._describe_io_as_lines() + progress_fp.write("# IO table\n" + "\n".join(table_lines) + "\n\n") + progress_fp.flush() + + for idx, block in enumerate(self.blocks, 1): + # Override the block’s load_checkpoint flag if requested + if self.force_checkpoint is not None: + block.load_checkpoint = self.force_checkpoint + + if self.progress: + print(f"▶ [{idx}/{total}] {block.tag} …", flush=True) + + t0 = time.perf_counter() + + # Capture output and trace exceptions + if self.capture_output: + buf_out, buf_err = io.StringIO(), io.StringIO() + try: + with contextlib.redirect_stdout(buf_out), contextlib.redirect_stderr(buf_err): + self.bundle = block(self.bundle) + except Exception: + buf_err.write(f"⚠️ Exception in block {block.tag}:\n") + traceback.print_exc(file=buf_err) + captured = buf_out.getvalue() + buf_err.getvalue() + # Write error log and re-raise + ts = _dt.datetime.now().strftime("%Y%m%d_%H%M%S") + (log_dir / f"{idx:02d}_{block.tag}_{ts}.log").write_text(captured, encoding="utf-8") + raise + else: + captured = buf_out.getvalue() + buf_err.getvalue() + if self.capture_output == "memory": + self.block_logs[block.tag] = captured + else: + ts = _dt.datetime.now().strftime("%Y%m%d_%H%M%S") + (log_dir / f"{idx:02d}_{block.tag}_{ts}.log").write_text(captured, encoding="utf-8") + else: + try: + self.bundle = block(self.bundle) + except Exception: + print(f"⚠️ Exception in block {block.tag}:") + traceback.print_exc() + raise + + elapsed = time.perf_counter() - t0 + if self.progress: + print(f"✓ [{idx}/{total}] {block.tag} finished in {elapsed:,.2f}s") + + if progress_fp: + ts_iso = _dt.datetime.now().isoformat(timespec="seconds") + progress_fp.write(f"{ts_iso}\t{idx}/{total}\t{block.tag}\t{elapsed:.2f}s\n") + progress_fp.flush() + + if progress_fp: + progress_fp.close() + return self.bundle + + # ------------------------------------------------------------------ # + # helper – produce describe-io lines without printing # + # ------------------------------------------------------------------ # + def _describe_io_as_lines(self) -> List[str]: + """ + Dynamically compute column widths based on the longest entry in each column + and return formatted lines describing each block's I/O. + """ + header_col1 = "Block (tag)" + header_col2 = "Needs" + header_col3 = "Provides" + + rows: List[Tuple[str, str, str]] = [] + for blk in self.blocks: + col1 = f"{blk.__class__.__name__} ({blk.tag})" + eff_needs = list(blk.needs) + [k for k, _ in blk.conditional_needs] + col2 = ", ".join(eff_needs) + col3 = str(list(blk.provides)) + rows.append((col1, col2, col3)) + + col1_width = max(len(header_col1), *(len(r[0]) for r in rows)) + col2_width = max(len(header_col2), *(len(r[1]) for r in rows)) + + header = f"{header_col1:<{col1_width}} │ {header_col2:<{col2_width}} │ {header_col3}" + separator = "─" * len(header) + + lines = [header, separator] + for col1, col2, col3 in rows: + lines.append(f"{col1:<{col1_width}} │ {col2:<{col2_width}} │ {col3}") + + return lines + + # ------------------------------------------------------------------ # + # user-friendly descriptions # + # ------------------------------------------------------------------ # + def describe_io(self) -> List[Tuple[str, Sequence[str], Sequence[str]]]: + """ + Print (and return) a table + + Block (tag) │ Needs (✓/✗) │ Provides + + Needs that are NOT currently satisfied are coloured red and suffixed + with a short reason. The logic is shared with check_io_consistency() + so both views stay consistent. + """ + rows: List[Tuple[str, Sequence[str], Sequence[str]]] = [] + + # preload bundle state + generic_seen: Set[str] = set() + by_tag: Dict[str, Set[str]] = {} + for base, bucket in self.bundle._store.items(): # type: ignore[attr-defined] + generic_seen.add(base) + for tag in bucket: + if tag != "_latest": + by_tag.setdefault(tag, set()).add(base) + + # collect rows with original and colored needs + for blk in self.blocks: + active_cond = [k for k, cond in blk.conditional_needs if cond(self.bundle, blk)] + eff_needs = list(blk.needs) + active_cond + + display_needs: List[str] = [] + canon = blk._canonical_needs + + for need in eff_needs: + if "." in need: + tag, key = need.split(".", 1) + if key not in by_tag.get(tag, set()): + display_needs.append(f"\x1b[31m{need} (bad namespace)\x1b[0m") + else: + display_needs.append(need) + else: + if need not in generic_seen: + display_needs.append(f"\x1b[31m{need} (missing)\x1b[0m") + else: + display_needs.append(need) + + # wrong-order annotation + suffixes = [n.split(".", 1)[-1] for n in eff_needs if n.split(".", 1)[-1] in canon] + if tuple(suffixes) != canon: + display_needs.append("\x1b[31m(order❌)\x1b[0m") + + rows.append((f"{blk.__class__.__name__} ({blk.tag})", display_needs, list(blk.provides))) + + # register provides + for p in blk.provides: + generic_seen.add(p) + by_tag.setdefault(blk.tag, set()).add(p) + + # prepare colored and plain needs strings + ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + colored_needs = [", ".join(needs) for _, needs, _ in rows] + plain_needs = [ANSI_RE.sub("", s) for s in colored_needs] + + # compute dynamic column widths + col1_label = "Block (tag)" + col2_label = "Needs (✓/✗)" + col1_width = max(len(col1_label), *(len(name) for name, _, _ in rows)) + col2_width = max(len(col2_label), *(len(s) for s in plain_needs)) + + # print if verbose + if self.verbose: + header = f"{col1_label:<{col1_width}} │ {col2_label:<{col2_width}} │ Provides" + print(header) + print("─" * len(header)) + + def ansi_ljust(s: str, width: int) -> str: + """ + Left-justify a string containing ANSI codes based on its visible length. + """ + visible = ANSI_RE.sub("", s) + padding = width - len(visible) + return s + " " * max(0, padding) + + for name, needs_list, provides_list in rows: + colored = ", ".join(needs_list) + field = ansi_ljust(colored, col2_width) + print(f"{name:<{col1_width}} │ {field} │ {provides_list}") + print() + + return rows + + # --------------------------------------------------------------------- # + # consistency check # + # --------------------------------------------------------------------- # + def check_io_consistency(self) -> None: + """See README – validates namespace, order, and conditional needs.""" + # --- notebook? --------------------------------------------------------- + try: + from IPython import get_ipython + IN_NOTEBOOK = "IPKernelApp" in get_ipython().config # type: ignore[attr-defined] + except Exception: + IN_NOTEBOOK = False + + # --- gather what we already have in the bundle ------------------------- + generic_seen: Set[str] = set() + by_tag: Dict[str, Set[str]] = {} # tag → {keys} + + for base, bucket in self.bundle._store.items(): # type: ignore[attr-defined] + generic_seen.add(base) + for tag, _ in bucket.items(): + if tag != "_latest": + by_tag.setdefault(tag, set()).add(base) + + # --- walk through the pipeline ---------------------------------------- + lines: list[str] = [] + for blk in self.blocks: + # evaluate conditional needs on the CURRENT bundle snapshot + active_conditional: list[str] = [] + for key, cond in blk.conditional_needs: + try: + if cond(self.bundle, blk): + active_conditional.append(key) + except Exception: + active_conditional.append(key) + + eff_needs = list(blk.needs) + active_conditional + + # ---------- rule 1 & 2 : missing / bad namespace ------------------ + missing, bad_ns = [], [] + for need in eff_needs: + if "." in need: + tag, key = need.split(".", 1) + if tag not in by_tag: + missing.append(need) + elif key not in by_tag[tag]: + bad_ns.append(need) + else: + if need not in generic_seen: + missing.append(need) + + # ---------- rule 3 : order of canonical needs --------------------- + canon = blk._canonical_needs + user_suffix_seq = [ + (n.split(".", 1)[1] if "." in n else n) + for n in eff_needs + if (n.split(".", 1)[-1]) in canon + ] + order_error = None + if tuple(user_suffix_seq) != canon: + order_error = ( + "wrong order – expected " + f"({', '.join(canon)}) but got " + f"({', '.join(user_suffix_seq) or '∅'})" + ) + + # ---------- compose message --------------------------------------- + problems: list[str] = [] + if missing: + problems.append( + "missing " + ", ".join(f"{m}" for m in missing) + ) + if bad_ns: + problems.append( + "bad namespace " + ", ".join(f"{b}" for b in bad_ns) + ) + if order_error: + problems.append(order_error) + + if problems: + msg = f"{blk.tag} – " + "; ".join(problems) + else: + msg = f"{blk.tag} – all needs met" + + lines.append(msg) + + # ---------- register provides for downstream blocks --------------- + for p in blk.provides: + generic_seen.add(p) + by_tag.setdefault(blk.tag, set()).add(p) + + # --- show report ------------------------------------------------------ + report_html = "
".join(lines) + if IN_NOTEBOOK: + display(HTML(report_html)) + elif self.verbose: + for line in lines: + print( + line.replace("", "") + .replace("", "") + .replace("", "") + .replace("", "") + .replace("", "") + ) + + # --------------------------------------------------------------------- # + # persistence helpers # + # --------------------------------------------------------------------- # + def save_settings(self) -> None: + """ + Serialize every block (via `jsonpickle`) into + ``/saved_settings/{idx}_{ClassName}.json``. + """ + saved_dir = Path(self.bundle["result_path"]) / "saved_settings" + saved_dir.mkdir(parents=True, exist_ok=True) + + for idx, blk in enumerate(self.blocks): + fn = saved_dir / f"{idx}_{blk.__class__.__name__}.json" + fn.write_text(jsonpickle.encode(blk), encoding="utf-8") + + def load_saved_settings(self) -> None: + """ + Load any JSON files in ``/saved_settings`` and restore + the state of blocks whose *class names* match. + """ + saved_dir = Path(self.bundle["result_path"]) / "saved_settings" + if not saved_dir.is_dir(): + raise FileNotFoundError(f"No saved settings found in {saved_dir!r}.") + + for fp in saved_dir.glob("*.json"): + _, class_name = fp.stem.split("_", 1) + for blk in self.blocks: + if blk.__class__.__name__ == class_name: + blk.load_settings(fp) + break + + self.check_io_consistency() # re-validate after loading + + # --------------------------------------------------------------------- # + # internal helpers # + # --------------------------------------------------------------------- # + def _assign_unique_tags(self) -> None: + """ + Ensure every block has a distinct `tag`. If duplicates are found, + they receive a numeric suffix (Block, Block2, Block3, …). + """ + counts: Dict[str, int] = {} + + for blk in self.blocks: + tag = getattr(blk, "tag", None) or blk.__class__.__name__ + counts[tag] = counts.get(tag, 0) + 1 + if counts[tag] > 1: + tag = f"{tag}{counts[tag]}" + blk.tag = tag # type: ignore[attr-defined] + + def _ensure_result_path(self) -> None: + """ + Guarantee the bundle has a writable `result_path`. + """ + if "result_path" not in self.bundle: + self.bundle["result_path"] = Path.cwd() / "results" diff --git a/TELF/pipeline/blocks/__init__.py b/TELF/pipeline/blocks/__init__.py new file mode 100644 index 00000000..c333ffb7 --- /dev/null +++ b/TELF/pipeline/blocks/__init__.py @@ -0,0 +1,58 @@ +from .base_block import AnimalBlock +from .vulture_clean_block import VultureCleanBlock +from .data_bundle import ( + DataBundle, + SAVE_DIR_BUNDLE_KEY, + DIR_LIST_BUNDLE_KEY, + RESULTS_DEFAULT, + SOURCE_DIR_BUNDLE_KEY +) +from .beaver_vocab_block import BeaverVocabBlock +from .orca_block import OrcaBlock +from .function_block import FunctionBlock +from .cheetah_filter_block import CheetahFilterBlock +from .hnmfk_block import HNMFkBlock +from .semantic_hnmfk_block import SemanticHNMFkBlock +from .nmfk_block import NMFkBlock +from .wolf_block import WolfBlock +from .scopus_block import ScopusBlock +from .s2_block import S2Block +from .auto_bunny_block import AutoBunnyBlock +from .beaver_doc_word_block import BeaverDocWordBlock +from .squirrel_block import SquirrelBlock +from .beaver_codependency_matrix_block import CodependencyMatrixBlock +from .dataframe_load_block import LoadDfBlock +from .field_filter_block import FilterByCountBlock +from .artic_fox_block import ArticFoxBlock +from .fox_block import FoxBlock +from .lmf_block import LMFBlock +from .split_block import SPLITBlock +from .split_transfer_block import SPLITTransferBlock +from .wordcloud_block import WordCloudBlock +from .load_terms_block import LoadTermsBlock +from .term_search_block import TermSearchBlock +from .term_attribution_block import TermAttributionBlock +from .scores_block import ComputeScoresBlock +from .sample_matrix_mask_block import SampleMatrixMaskBlock +from .clean_duplicates_block import CleanDuplicatesBlock +from .merge_scopus_s2_block import MergeScopusS2Block +from .clean_affiliations_block import CleanAffiliationsBlock +from .sbatch_block import SBatchBlock +from .beaver_cooccurrence_block import BeaverCooccurrenceBlock +from .beaver_something_words_block import BeaverSomethingWordsBlock +from .beaver_something_words_time_block import BeaverSomethingWordsTimeBlock +from .beaver_cocitation_tensor_block import BeaverCocitationTensorBlock +from .beaver_coauthor_tensor_block import BeaverCoauthorTensorBlock +from .beaver_citation_tensor_block import BeaverCitationTensorBlock +from .beaver_participation_tensor_block import BeaverParticipationTensorBlock +from .rescalk_block import RESCALkBlock +from .combine_dir_loop_df_block import ConcatenateDFBlock +from .pipeline_summary_block import PipelineSummaryBlock +from .author_lines_block import ExpandAuthorAffiliationBlock +from .core_register_block import CoreRegister +from .uniqueness_block import UniquenessDFBlock + +from .group_summary_block import GroupSummaryBlock +from .post_process_cluster_analysis_block import ClusteringAnalyzerBlock +from .post_process_label_analysis_block import LabelAnalyzerBlock +from .peacock_stats_block import PeacockStatsBlock \ No newline at end of file diff --git a/TELF/pipeline/blocks/artic_fox_block.py b/TELF/pipeline/blocks/artic_fox_block.py new file mode 100644 index 00000000..160915f5 --- /dev/null +++ b/TELF/pipeline/blocks/artic_fox_block.py @@ -0,0 +1,67 @@ +from pathlib import Path +from typing import Dict, Sequence, Any, Tuple +from .base_block import AnimalBlock +from .data_bundle import DataBundle + +from ...post_processing import ArcticFox +from ...factorization import HNMFk + +class ArticFoxBlock(AnimalBlock): + + CANONICAL_NEEDS = ("df", 'vocabulary', "model_path",) + + def __init__( + self, + col: str = "clean_title_abstract", + needs = CANONICAL_NEEDS, + provides = ("block_status",), + tag: str = "ArticFox", + *, + init_settings: Dict[str, Any] = None, + call_settings: Dict[str, Any] = None, + **kw, + ) -> None: + + self.col = col + default_init = { + 'clean_cols_name': self.col, + 'embedding_model': "SCINCL", + } + default_call = { + 'ollama_model': "llama3.2:3b-instruct-fp16", # Language model used for semantic label generation + 'label_clusters': True, # Enable automatic labeling of clusters + 'generate_stats': True, # Generate cluster-level statistics + 'process_parents': True, # Propagate labels or stats upward through the hierarchy + 'skip_completed': True, # Skip processing of nodes already labeled/stored + 'label_criteria': { # Rules to filter generated labels + "minimum words": 2, + "maximum words": 6 + }, + 'label_info': { # Additional metadata to associate with generated labels + "source": "Science" + }, + 'number_of_labels': 5 # Number of candidate labels to generate per node + } + + super().__init__( + needs = needs, + provides = provides, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + tag=tag, + **kw, + ) + + def run(self, bundle: DataBundle) -> None: + df = self.load_path(bundle[self.needs[0]]) + vocabulary = self.load_path(bundle[self.needs[1]]) + model = HNMFk(experiment_name=bundle[self.needs[2]]) + model.load_model() # Loads model from the provided experiment_name path + pipeline = ArcticFox( + model=model, + **self.init_settings + ) + pipeline.run_full_pipeline(data_df = df, + vocab = vocabulary, + **self.call_settings) + bundle[f"{self.tag}.{self.provides[0]}"] = "Done" \ No newline at end of file diff --git a/TELF/pipeline/blocks/author_lines_block.py b/TELF/pipeline/blocks/author_lines_block.py new file mode 100644 index 00000000..95818133 --- /dev/null +++ b/TELF/pipeline/blocks/author_lines_block.py @@ -0,0 +1,126 @@ +# blocks/expand_and_assign_block.py + +from __future__ import annotations +from pathlib import Path +from typing import Dict, Sequence, Any +import pandas as pd +import ast + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +class ExpandAuthorAffiliationBlock(AnimalBlock): + """ + needs : ('df',) + provides : ('df',) – expanded DataFrame with *_component columns + tag : 'ExpandAssign' + """ + CANONICAL_NEEDS = ('df',) + + # columns to drop after expansion + _TO_DROP = [ + 'author_ids','authors','affiliations', + 'countries','query','affiliation_names', + 'slic_affiliations','slic_author_ids','type', + 'affiliation_ids','slic_authors','s2_authors', + 's2_author_ids', + ] + + def __init__(self, tag='ExpandAssign', **kw) -> None: + super().__init__( + needs=self.CANONICAL_NEEDS, + provides=('author_df',), + tag=tag, + **kw, + ) + + def run(self, bundle: DataBundle) -> None: + # ── 1) LOAD & EXPAND ──────────────────────────────────── + src = bundle[self.needs[0]] + if isinstance(src, (str, Path)): + df0 = pd.read_csv(src, dtype=str) + else: + df0 = src.copy().astype(str) + + # drop rows missing both IDs + if {'author_ids','authors'}.issubset(df0.columns): + df0 = df0.dropna(subset=['author_ids','authors'], how='all') + + records: list[Dict[str,Any]] = [] + for _, row in df0.iterrows(): + base = row.drop(['author_ids','authors','affiliations'], errors='ignore').to_dict() + + # build id→name map + ids = [i.strip() for i in row['author_ids'].split(';') if i.strip()] + names = [n.strip() for n in row['authors'].split(';')] + auth_map = dict(zip(ids, names)) + + # parse affiliations JSON if present + raw_aff = row.get('affiliations') + try: + affs = ast.literal_eval(raw_aff) if pd.notna(raw_aff) else {} + except Exception: + affs = {} + + if isinstance(affs, dict) and affs: + # one row per (affiliation × author) + for aff_id, aff in affs.items(): + if not isinstance(aff, dict): + continue + for aid in aff.get('authors', []): + aid = aid.strip() + records.append({ + **base, + 'author_id': aid, + 'author': auth_map.get(aid, ''), + 'affiliation_id': aff_id, + 'affiliation': aff.get('name'), + 'country': aff.get('country'), + }) + else: + # no affiliations → one row per author + for aid, aname in auth_map.items(): + records.append({ **base, 'author_id': aid, 'author': aname }) + + expanded_df = pd.DataFrame(records) + + # drop everything in one call + drop_cols = [c for c in self._TO_DROP if c in expanded_df.columns] + if drop_cols: + expanded_df = expanded_df.drop(columns=drop_cols) + + # ── 2) AUTO-DISCOVER & ASSIGN COMPONENTS ───────────────── + root = Path(bundle[SAVE_DIR_BUNDLE_KEY]) + # map of “column_name” → { eid: component_id, … } + comp_maps: Dict[str, Dict[str,str]] = {} + + # find every CSV under any Wolf* folder + for csv_path in root.rglob("*component*documents*.csv"): + rel = csv_path.relative_to(root).parts + # expect: ( WolfTag, category, comp_id, filename ) + if len(rel) < 3 or not rel[1] or not rel[2].isdigit(): + continue + + category, comp_id = rel[1], rel[2] + col = f"{category.replace('-', '')}_component" + comp_dict = comp_maps.setdefault(col, {}) + + # read only eid column + tmp = pd.read_csv(csv_path, usecols=['eid'], dtype=str) + for eid in tmp['eid'].dropna().astype(str): + comp_dict[eid] = comp_id + + # now vectorized‐map each column + for col, mapping in comp_maps.items(): + expanded_df[col] = expanded_df['eid'].astype(str).map(mapping).astype(pd.StringDtype()) + + # ── 3) SAVE & CHECKPOINT ─────────────────────────────── + if SAVE_DIR_BUNDLE_KEY in bundle: + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(parents=True, exist_ok=True) + out_csv = out_dir / "expanded_with_components.csv" + expanded_df.to_csv(out_csv, index=False, encoding="utf-8-sig") + self.register_checkpoint(self.provides[0], out_csv) + + # ── 4) PUSH BACK INTO BUNDLE ──────────────────────────── + bundle[f"{self.tag}.{self.provides[0]}"] = expanded_df diff --git a/TELF/pipeline/blocks/auto_bunny_block.py b/TELF/pipeline/blocks/auto_bunny_block.py new file mode 100644 index 00000000..0b8d587a --- /dev/null +++ b/TELF/pipeline/blocks/auto_bunny_block.py @@ -0,0 +1,130 @@ +from pathlib import Path +from typing import Any, Dict, Sequence, Optional, Callable, Tuple + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +from TELF.pre_processing.Vulture.modules.substitute import SubstitutionCleaner +from TELF.pipeline.blocks import VultureCleanBlock +from TELF.helpers.terms import resolve_substitution_conflicts +from TELF.applications.Bunny import AutoBunny, AutoBunnyStep + + +class AutoBunnyBlock(AnimalBlock): + CANONICAL_NEEDS = ('df','terms',) + + def __init__( + self, + num_hops: int = 5, + cheetah_settings: Dict = { + 'in_title': False, + 'in_abstract': True, + 'and_search': False, + }, + modes: Sequence[str]=['citations'], + hop_priority: str= "frequency", + needs: Tuple[str, str] = CANONICAL_NEEDS, + provides: Tuple[str, ...] = ('df',), + use_substitutions:bool=False, + use_vulture_steps:bool=False, + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "AutoBunny", + *, + init_settings: Optional[Dict[str, Any]] = None, + call_settings: Optional[Dict[str, Any]] = None, + verbose: bool = True, + **kw: Any, + ) -> None: + """ + Dataset expansion tool + + ───────────────────────────────────────────────────────────── + always needs : ('df', 'terms') + needs 'substitutions' only when use_substitutions=True and use_vulture_steps=False + needs 'vulture_steps' only when use_vulture_steps=True + provides : ('df',) + tag : 'AutoBunny' (namespace for its outputs) + """ + + # Default init settings for AutoBunny + default_init: Dict[str, Any] = { + 's2_key': None, + 'scopus_keys': None, + 'cache_dir': None, + 'verbose': True, + } + self.use_substitutions = use_substitutions + self.use_vulture_steps = use_vulture_steps + + conds = list(conditional_needs or []) + if self.use_substitutions and not self.use_vulture_steps: + conds.append(("substitutions", lambda _b, _s: True)) + + if self.use_vulture_steps: + conds.append(("vulture_steps", lambda _b, _s: True)) + self.VULTURE_STEPS = None + else: + # Load the standard Vulture steps as a list + self.VULTURE_STEPS = list(VultureCleanBlock().steps) + + # Placeholder for Cheetah settings (unused here) + self.cheetah_settings = cheetah_settings + + # Build initial AutoBunnyStep list + steps = [ + AutoBunnyStep( + modes=modes, + max_papers=0, + hop_priority=hop_priority, + vulture_settings=self.VULTURE_STEPS, + ) + for _ in range(num_hops) + ] + + default_call: Dict[str, Any] = { + 'steps': steps, + 'filter_type': 'AFFILCOUNTRY', + 'filter_value': None, + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conds, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kw, + ) + + def run(self, bundle: DataBundle) -> None: + df = self.load_path(bundle[self.needs[0]]) + terms = self.load_path(bundle[self.needs[1]]) + self.cheetah_settings['query'] = terms + + if self.use_vulture_steps: + self.VULTURE_STEPS = self.load_path(bundle["vulture_steps"]) + + for step in self.call_settings['steps']: + step.vulture_settings = self.VULTURE_STEPS + step.cheetah_settings = self.cheetah_settings + + elif self.use_substitutions and not self.use_vulture_steps: + substitutions = self.load_path(bundle["vulture_steps"]) + initial_sub = SubstitutionCleaner(substitutions, permute=True, lower=True, lemmatize=True) + final_sub = SubstitutionCleaner(substitutions, permute=False, lower=False, lemmatize=True) + vchain = [initial_sub] + self.VULTURE_STEPS + [final_sub] + + for step in self.call_settings['steps']: + step.vulture_settings = vchain + step.cheetah_settings = self.cheetah_settings + + + self.output_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + self.output_dir.mkdir(parents=True, exist_ok=True) + + ab = AutoBunny(core=df, output_dir=self.output_dir, **self.init_settings) + ab_df = ab.run(**self.call_settings) + + bundle[f"{self.tag}.{self.provides[0]}"] = ab_df diff --git a/TELF/pipeline/blocks/base_block.py b/TELF/pipeline/blocks/base_block.py new file mode 100644 index 00000000..81d2c96e --- /dev/null +++ b/TELF/pipeline/blocks/base_block.py @@ -0,0 +1,323 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Sequence, Callable, Iterable, Tuple, Any, Dict, Union +import copy, jsonpickle, json +import numpy as np, pandas as pd, scipy.sparse as sp +from PIL import Image +import pickle +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +Cond = Callable[[DataBundle, "AnimalBlock"], bool] + +class AnimalBlock(ABC): + """ + Base-class for every pipeline block. + + Each concrete subclass must define + + • needs – immutable prerequisites (tuple[str, ...]) + • provides – keys it guarantees to add (tuple[str, ...]) + • conditional_needs – list[ (key, condition(bundle, self)) ] + """ + _CKPT_FILE = "__checkpoints__.json" + + # ────────────────────────────────────────────────────────────────── + # constructor + # ────────────────────────────────────────────────────────────────── + def __init__( + self, + *, + needs: Sequence[str] = (), + provides: Sequence[str] = (), + conditional_needs: Sequence[Tuple[str, Cond]] = (), + checkpoint_keys: Sequence[str] | None = None, + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + tag: str | None = None, + verbose: bool = True, + checkpoint: bool = True, + load_checkpoint: bool = True, + **attrs: Any, + ): + self.needs: Tuple[str, ...] = tuple(needs) + self.provides: Tuple[str, ...] = tuple(provides) + self.conditional_needs = list(conditional_needs) + self.init_settings = init_settings or {} + self.call_settings = call_settings or {} + self.tag: str = tag or self.__class__.__name__ + self.verbose = bool(verbose) + self.checkpoint = bool(checkpoint) + self.load_checkpoint = bool(load_checkpoint) + + self._ckpt_keys = set(checkpoint_keys) if checkpoint_keys else set(self.provides) + + if hasattr(self.__class__, "CANONICAL_NEEDS"): + self._canonical_needs = tuple(self.__class__.CANONICAL_NEEDS) + else: + self._canonical_needs = tuple(n.split(".", 1)[-1] for n in self.needs) + + for k, v in attrs.items(): + setattr(self, k, v) + + if self.verbose: + runtime_needs = list(self.needs) + [k for k, _ in self.conditional_needs] + print(f"[{self.tag}] needs → ({', '.join(runtime_needs) or '∅'})" + f" provides → ({', '.join(self.provides) or '∅'})") + + # ─────────────────────── checkpoint helpers ─────────────────────── + def _ckpt_dir(self, bundle: DataBundle) -> Path: + return Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + def _ckpt_path(self, bundle: DataBundle) -> Path: + return self._ckpt_dir(bundle) / self._CKPT_FILE + + def _load_ckpt_map(self, bundle: DataBundle) -> Dict[str, str]: + fp = self._ckpt_path(bundle) + if fp.is_file(): + try: + return json.loads(fp.read_text(encoding="utf-8")) + except Exception: + pass + return {} + + def _save_ckpt_map(self, bundle: DataBundle, ckpt_map: Dict[str, str]) -> None: + if not self.checkpoint: + return + fp = self._ckpt_path(bundle) + fp.parent.mkdir(parents=True, exist_ok=True) + fp.write_text(json.dumps(ckpt_map, indent=2), encoding="utf-8") + + def register_checkpoint(self, bundle_key: str, path: Union[str, Path]) -> None: + """ + Call inside `run()` for every provide you persist to disk. + + Example + ------- + final_csv = outdir / "results.csv" + df.to_csv(final_csv, index=False) + self.register_checkpoint("df", final_csv) + """ + path = str(path) + self._pending_ckpt_map[bundle_key] = path + + @staticmethod + def _merge(default: Dict[str, Any], override: Dict[str, Any] | None): + return {**default, **(override or {})} + + # ───────────────────────────── run cycle ─────────────────────────── + @abstractmethod + def run(self, bundle: DataBundle) -> None: + """ + Mutate the bundle in-place; do not return. + """ + ... + + def __call__(self, bundle: DataBundle) -> DataBundle: + # -------------------------------------------------------------- + # 0) maybe load from checkpoint and bail early + # -------------------------------------------------------------- + if self.load_checkpoint: + ckpt = self._load_ckpt_map(bundle) + have = self._ckpt_keys.issubset(ckpt.keys()) + exist = have and all(Path(ckpt[k]).is_file() for k in self._ckpt_keys) + if exist: + # hydrate only the keys we cached + for k in self._ckpt_keys: + bundle[f"{self.tag}.{k}"] = self.load_path(ckpt[k]) + + # load any 'provides' from the class that is not saved to disk + self._after_checkpoint_skip(bundle) + + if self.verbose: + print(f"[{self.tag}] ✔ loaded from checkpoint") + # ensure *all* provides are present (lazy ones may already + # be in the bundle from a previous block run) + return bundle + + # -------------------------------------------------------------- + # 1) normal validation of needs + # -------------------------------------------------------------- + runtime_needs = set(self.needs) + for key, cond in self.conditional_needs: + if cond(bundle, self): + runtime_needs.add(key) + missing = [k for k in runtime_needs if k not in bundle] + if missing: + raise KeyError(f"{self.tag}: missing required keys {missing}") + + # -------------------------------------------------------------- + # 2) run block + # -------------------------------------------------------------- + self._pending_ckpt_map: Dict[str, str] = {} + self.run(bundle) + + # -------------------------------------------------------------- + # 3) verify outputs + # -------------------------------------------------------------- + view = bundle.namespaced(self.tag) + missing_out = [k for k in self.provides if k not in view] + if missing_out: + raise KeyError(f"{self.tag}: failed to provide {missing_out}") + + # -------------------------------------------------------------- + # 4) persist checkpoints (only those you registered) + # -------------------------------------------------------------- + if self.checkpoint: + if self._ckpt_keys.issubset(self._pending_ckpt_map): + self._save_ckpt_map(bundle, { + k: self._pending_ckpt_map[k] for k in self._ckpt_keys + }) + if self.verbose: + print(f"[{self.tag}] ⭳ checkpoint saved") + + return bundle + + def _after_checkpoint_skip(self, bundle: DataBundle) -> None: + # no-op: do nothing by default + pass + + def save_settings(self, path: Union[str, Path]) -> None: + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + serialized = jsonpickle.encode(self, indent=2) + p.write_text(serialized, encoding="utf-8") + + def load_settings(self, path: Union[str, Path]) -> None: + p = Path(path) + raw = p.read_text(encoding="utf-8") + state = jsonpickle.decode(raw) + if not hasattr(state, '__dict__'): + raise ValueError(f"Bad settings format: expected object, got {type(state)}") + self.__dict__.update(state.__dict__) + self.settings = state + + def print_settings(self) -> None: + for key, val in (vars(self).items()): + print(f"{key}: {val!r}") + + print('needs') + for i in self.needs: + print(f"\t{i!r}") + + print('provides') + for i in self.provides: + print(f"\t{i!r}") + + + def load_path(self, path: Union[str, Path]) -> Any: + # In-memory objects pass straight through + if type(path) in [pd.DataFrame, sp._csr.csr_matrix, np.ndarray, dict, list]: + return path + + p = Path(path) + if not p.exists(): + raise FileNotFoundError(f"File not found: {p}") + + ext = p.suffix.lower() + + if ext in {".csv", ".tsv"}: + return pd.read_csv(p) + + if ext == ".json": + return json.loads(p.read_text(encoding="utf-8")) + + if ext in {".txt", ".html", ".htm"}: + return p.read_text(encoding="utf-8") + + if ext == ".npy": + return np.load(p, allow_pickle=True) + + if ext == ".npz": + try: + return sp.load_npz(str(p)) + except Exception: + return np.load(p, allow_pickle=True) + + # ─── Pickle / gpickle ────────────────────────────────────── + if ext in {".gpickle", ".pkl", ".pickle"}: + with p.open("rb") as f: + return pickle.load(f) + + if ext in {".png", ".jpg", ".jpeg", ".gif", ".bmp"}: + return Image.open(p) + + # fallback to raw bytes + return p.read_bytes() + + + def save_path(self, data: Any, path: Union[str, Path]) -> None: + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + ext = p.suffix.lower() + + # ─── Pickle / gpickle ────────────────────────────────────── + if ext in {".gpickle", ".pkl", ".pickle"}: + with p.open("wb") as f: + pickle.dump(data, f) + return + + # ─── Common data types ───────────────────────────────────── + if isinstance(data, pd.DataFrame): + data.to_csv(p, index=False) + return + + if isinstance(data, (dict, list)): + with p.open("w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + return + + if isinstance(data, (str, int, float)): + with p.open("w", encoding="utf-8") as f: + f.write(str(data)) + return + + if isinstance(data, np.ndarray): + if ext in {".csv", ".txt"}: + np.savetxt(p, data, delimiter=",") + else: + np.save(str(p), data) + return + + if sp.issparse(data): + sp.save_npz(str(p), data) + return + + if isinstance(data, Image.Image): + data.save(p) + return + + if isinstance(data, (bytes, bytearray)): + with p.open("wb") as f: + f.write(data) + return + + raise ValueError(f"Unsupported data type: {type(data)}") + + + def copy(instance: Any, + needs: Tuple[str, ...]=None, + provides: Tuple[str, ...] = None) -> Any: + """ + Return a deep-copy of `instance` with updated `needs` and `provides`. + + Parameters + ---------- + instance + Any object that has `needs` and `provides` attributes. + new_needs + Tuple of strings to assign to the copied instance's `needs`. + new_provides + Tuple of strings to assign to the copied instance's `provides`. + + Returns + ------- + Any + A deepcopy of `instance` with its I/O replaced. + """ + new_obj = copy.deepcopy(instance) + if needs: + new_obj.needs = needs + if provides: + new_obj.provides = provides + return new_obj \ No newline at end of file diff --git a/TELF/pipeline/blocks/beaver_citation_tensor_block.py b/TELF/pipeline/blocks/beaver_citation_tensor_block.py new file mode 100644 index 00000000..2d2ae303 --- /dev/null +++ b/TELF/pipeline/blocks/beaver_citation_tensor_block.py @@ -0,0 +1,112 @@ +# blocks/beaver_citation_tensor_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import pandas as pd +from ...pre_processing.Beaver import Beaver +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class BeaverCitationTensorBlock(AnimalBlock): + """ + Build an **(authors × papers × references)** citation tensor with *Beaver*. + + The underlying call is ``beaver.citation_tensor`` which returns: + + * **X** – 3-mode tensor + * **authors** – unique author identifiers (axis 0) + * **paper_ids** – paper / EID identifiers (axis 1) + * **years** – publication years (axis 2 or metadata) + + ───────────────────────────────────────────────────────────── + needs : ('df',) + provides : ('X', 'authors', 'paper_ids', 'years') + tag : 'BeaverCit' + """ + CANONICAL_NEEDS = ("df",) + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + target_columns: Tuple[str, str, str, str] = ( + "author_ids", + "eid", + "references", + "year", + ), + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("X", "authors", "paper_ids", "years"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "BeaverCit", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + # settings that go to Beaver(...) + default_init = { + "n_jobs": -1, + "n_nodes": 1, + } + + # settings for beaver.citation_tensor(...) + default_call = { + "target_columns": target_columns, + "dimension_order": [0, 1, 2], + "split_authors_with": ";", + "split_references_with": ";", + "verbose": False, + "n_jobs": 1, + "n_nodes": 1, + # "joblib_backend": "multiprocessing", # disabled (see note) + # "save_path" injected at runtime if bundle['result_path'] exists + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # 1️⃣ Load / copy the DataFrame ------------------------------------ + src_df = bundle[self.needs[0]] + df = self.load_path(src_df) if isinstance(src_df, (str, Path)) else src_df.copy() + + # 2️⃣ Ensure rows contain all required columns --------------------- + for col in self.call_settings["target_columns"]: + if col in df.columns: + df = df.dropna(subset=[col]) + + # 3️⃣ Prepare call specification ----------------------------------- + cb: Dict[str, Any] = dict(self.call_settings) + + # dynamic `save_path` + if "save_path" not in cb and SAVE_DIR_BUNDLE_KEY in bundle: + cb["save_path"] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + # 4️⃣ Execute Beaver ------------------------------------------------- + beaver = Beaver(**self.init_settings) + X, authors, paper_ids, years = beaver.citation_tensor(dataset=df, **cb) + + # 5️⃣ Store results -------------------------------------------------- + ns = self.tag + bundle[f"{ns}.{self.provides[0]}"] = X + bundle[f"{ns}.{self.provides[1]}"] = authors + bundle[f"{ns}.{self.provides[2]}"] = paper_ids + bundle[f"{ns}.{self.provides[3]}"] = years diff --git a/TELF/pipeline/blocks/beaver_coauthor_tensor_block.py b/TELF/pipeline/blocks/beaver_coauthor_tensor_block.py new file mode 100644 index 00000000..8b54b354 --- /dev/null +++ b/TELF/pipeline/blocks/beaver_coauthor_tensor_block.py @@ -0,0 +1,102 @@ +# blocks/beaver_coauthor_tensor_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import pandas as pd +from ...pre_processing.Beaver import Beaver +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class BeaverCoauthorTensorBlock(AnimalBlock): + """ + Build an **(authors × authors × time)** co-author tensor with *Beaver*. + + The underlying call is ``beaver.coauthor_tensor`` which returns: + + * **X** – 3-mode tensor (author i × author j × year) + * **authors** – unique author identifiers (shared by modes 0 & 1) + * **years** – temporal index (e.g. publication years) + + ───────────────────────────────────────────────────────────── + needs : ('df',) + provides : ('X', 'authors', 'years') + tag : 'BeaverCoAuth' + """ + CANONICAL_NEEDS = ("df",) + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + target_columns: Tuple[str, str] = ("author_ids", "year"), + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("X", "authors", "years"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "BeaverCoAuth", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + # Parameters passed to Beaver(...) + default_init = { + "n_jobs": -1, + "n_nodes": 1, + } + + # Parameters forwarded to beaver.coauthor_tensor(...) + default_call = { + "target_columns": target_columns, + "split_authors_with": ";", + "verbose": False, + "n_jobs": -1, + "n_nodes": 1, + # "save_path" injected dynamically if bundle['result_path'] exists + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # 1️⃣ Get the dataframe -------------------------------------------- + src_df = bundle[self.needs[0]] + df = self.load_path(src_df) if isinstance(src_df, (str, Path)) else src_df.copy() + + # 2️⃣ Drop rows missing required columns ---------------------------- + for col in self.call_settings["target_columns"]: + if col in df.columns: + df = df.dropna(subset=[col]) + + # 3️⃣ Build call dictionary ----------------------------------------- + cb: Dict[str, Any] = dict(self.call_settings) + + # dynamic save_path + if "save_path" not in cb and SAVE_DIR_BUNDLE_KEY in bundle: + cb["save_path"] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + # 4️⃣ Execute Beaver ------------------------------------------------- + beaver = Beaver(**self.init_settings) + X, authors, years = beaver.coauthor_tensor(dataset=df, **cb) + + # 5️⃣ Store outputs -------------------------------------------------- + ns = self.tag + bundle[f"{ns}.{self.provides[0]}"] = X + bundle[f"{ns}.{self.provides[1]}"] = authors + bundle[f"{ns}.{self.provides[2]}"] = years diff --git a/TELF/pipeline/blocks/beaver_cocitation_tensor_block.py b/TELF/pipeline/blocks/beaver_cocitation_tensor_block.py new file mode 100644 index 00000000..c6b454b7 --- /dev/null +++ b/TELF/pipeline/blocks/beaver_cocitation_tensor_block.py @@ -0,0 +1,106 @@ +# blocks/beaver_cocitation_tensor_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import pandas as pd +from ...pre_processing.Beaver import Beaver +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class BeaverCocitationTensorBlock(AnimalBlock): + """ + Build a **co-citation** tensor with *Beaver*. + + The call ``beaver.cocitation_tensor`` returns + + * **X** – 3-mode tensor (author i × author j × year) + * **authors** – unique author IDs (shared by modes 0 & 1) + * **years** – temporal index (publication years) + + ───────────────────────────────────────────────────────────── + needs : ('df',) + provides : ('X', 'authors', 'years') + tag : 'BeaverCoCit' + """ + CANONICAL_NEEDS = ("df",) + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + target_columns: Tuple[str, str, str, str] = ( + "author_ids", + "year", + "eid", + "references", + ), + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("X", "authors", "years"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "BeaverCoCit", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + # arguments for Beaver(...) + default_init = { + "n_jobs": -1, + "n_nodes": 1, + } + + # arguments for beaver.cocitation_tensor(...) + default_call = { + "target_columns": target_columns, + "split_authors_with": ";", + "split_references_with": ";", + "verbose": False, + "n_jobs": 1, + "n_nodes": 1, + # "save_path" is injected dynamically if bundle[SAVE_DIR_BUNDLE_KEY] exists + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # 1️⃣ Load DataFrame ------------------------------------------------- + src_df = bundle[self.needs[0]] + df = self.load_path(src_df) if isinstance(src_df, (str, Path)) else src_df.copy() + + # 2️⃣ Ensure required columns are present & not null ---------------- + for col in self.call_settings["target_columns"]: + if col in df.columns: + df = df.dropna(subset=[col]) + + # 3️⃣ Build call-time configuration --------------------------------- + cb: Dict[str, Any] = dict(self.call_settings) + if "save_path" not in cb and SAVE_DIR_BUNDLE_KEY in bundle: + cb["save_path"] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + # 4️⃣ Execute Beaver ------------------------------------------------- + beaver = Beaver(**self.init_settings) + X, authors, years = beaver.cocitation_tensor(dataset=df, **cb) + + # 5️⃣ Store outputs in bundle --------------------------------------- + ns = self.tag + bundle[f"{ns}.{self.provides[0]}"] = X + bundle[f"{ns}.{self.provides[1]}"] = authors + bundle[f"{ns}.{self.provides[2]}"] = years diff --git a/TELF/pipeline/blocks/beaver_codependency_matrix_block.py b/TELF/pipeline/blocks/beaver_codependency_matrix_block.py new file mode 100644 index 00000000..67be7718 --- /dev/null +++ b/TELF/pipeline/blocks/beaver_codependency_matrix_block.py @@ -0,0 +1,91 @@ +# blocks/codependency_matrix_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import os, sparse +from ...pre_processing import Beaver +from ...helpers.file_system import load_file_as_dict + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class CodependencyMatrixBlock(AnimalBlock): + """ + Build a 3-mode author–year tensor and flatten it to a co-authorship + matrix + node-ID map. + + ───────────────────────────────────────────────────────────── + needs : ('df',) + provides : ('X', 'node_ids') + tag : 'CodeMatrix' + """ + CANONICAL_NEEDS = ('df', ) + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + col: str = "slic_author_ids", + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("X", "node_ids"), + conditional_needs: Sequence[Tuple[str, Any]] = (), # none for now + tag: str = "CodeMatrix", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + self.col = col # store the column name + + default_init = {} + default_call = { + "target_columns": [self.col, "year"], + "split_authors_with": ";", + "verbose": True, + "n_jobs": -1, + "authors_idx_map": {}, + "joblib_backend": "threading", + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # paths + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / "CodependencyMatrixBlock" / self.col + out_dir.mkdir(parents=True, exist_ok=True) + + # dataframe + df = bundle[self.needs[0]].copy() + + # build tensor with Beaver + beaver = Beaver(**self.init_settings) + cfg = dict(self.call_settings) + cfg.update({"dataset": df, "target_columns": [self.col, "year"], "save_path": out_dir}) + + beaver.coauthor_tensor(**cfg) + + # load results + X = sparse.load_npz(out_dir / "coauthor.npz").sum(axis=2) # flatten 3-mode tensor + node_ids = load_file_as_dict(out_dir / "Authors.txt") + + # write back under this block’s namespace + bundle[f"{self.tag}.{self.provides[0]}"] = X + bundle[f"{self.tag}.{self.provides[1]}"] = node_ids diff --git a/TELF/pipeline/blocks/beaver_cooccurrence_block.py b/TELF/pipeline/blocks/beaver_cooccurrence_block.py new file mode 100644 index 00000000..16d2b42c --- /dev/null +++ b/TELF/pipeline/blocks/beaver_cooccurrence_block.py @@ -0,0 +1,107 @@ +# blocks/beaver_cooccurrence_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import pandas as pd +from ...pre_processing.Beaver import Beaver +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class BeaverCooccurrenceBlock(AnimalBlock): + """ + Build a **co-occurrence** (and SPPMI) matrix with *Beaver*. + + ───────────────────────────────────────────────────────────── + needs : ('df', 'vocabulary') + provides : ('cooccurrence', 'sppmi') + tag : 'BeaverCooc' + """ + CANONICAL_NEEDS = ("df", "vocabulary") + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + target_column: str = "clean_abstract", + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("cooccurrence", "sppmi"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "BeaverCooc", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + # settings sent to Beaver(...) + default_init = { + "n_jobs": -1, + "n_nodes": 1, + } + + # settings sent to beaver.cooccurrence_matrix(...) + default_call = { + "target_column": target_column, + "cooccurrence_settings": { + "n_jobs": 2, + "window_size": 100, + # 'vocabulary' injected at runtime + }, + "sppmi_settings": {}, + # "save_path" will be added automatically if bundle[SAVE_DIR_BUNDLE_KEY] exists + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # 1️⃣ Fetch dataframe ---------------------------------------------- + src_df = bundle[self.needs[0]] + if isinstance(src_df, (str, Path)): + df = self.load_path(src_df) + else: + df = src_df.copy() + + # 2️⃣ Fetch vocabulary -------------------------------------------- + vocabulary = bundle[self.needs[1]] + + # 3️⃣ Drop rows without the target column -------------------------- + target_col = self.call_settings["target_column"] + df = df.dropna(subset=[target_col]) + + # 4️⃣ Prepare Beaver & call spec ----------------------------------- + beaver = Beaver(**self.init_settings) + + cb: Dict[str, Any] = dict(self.call_settings) + + # inject vocabulary into nested dict + cb.setdefault("cooccurrence_settings", {}) + cb["cooccurrence_settings"]["vocabulary"] = vocabulary + + # dynamic save path + if "save_path" not in cb and SAVE_DIR_BUNDLE_KEY in bundle: + cb["save_path"] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + # 5️⃣ Build matrices ------------------------------------------------ + cooc_mat, sppmi_mat = beaver.cooccurrence_matrix(dataset=df, **cb) + + # 6️⃣ Store outputs under namespace -------------------------------- + ns = self.tag + bundle[f"{ns}.{self.provides[0]}"] = cooc_mat # 'cooccurrence' + bundle[f"{ns}.{self.provides[1]}"] = sppmi_mat # 'sppmi' diff --git a/TELF/pipeline/blocks/beaver_doc_word_block.py b/TELF/pipeline/blocks/beaver_doc_word_block.py new file mode 100644 index 00000000..93a56215 --- /dev/null +++ b/TELF/pipeline/blocks/beaver_doc_word_block.py @@ -0,0 +1,94 @@ +# blocks/beaver_docword_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Dict, Sequence, Any + +from ...pre_processing.Beaver import Beaver +from .base_block import AnimalBlock +from .data_bundle import DataBundle,SAVE_DIR_BUNDLE_KEY + + +class BeaverDocWordBlock(AnimalBlock): + """ + Build a sparse **document-word** matrix with *Beaver*. + + ───────────────────────────────────────────────────────────── + needs : ('df', 'vocabulary') + provides : ('X',) – the matrix + tag : 'BeaverDW' + """ + CANONICAL_NEEDS = ("df", "vocabulary", ) + + def __init__( + self, + *, + col: str = "clean_title_abstract", + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("X",), + conditional_needs: Sequence[tuple[str, Any]] = (), # none for now + tag: str = "BeaverDW", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + **kw, + ) -> None: + + self.col = col + default_init = { + "n_jobs": -1, + "n_nodes": 1, + } + + default_call = { + 'target_column':self.col, + 'options':{"min_df": 5, "max_df": 0.5}, + 'highlighting':[], + 'weights':[], + 'matrix_type':"tfidf", + 'verbose':False, + 'return_object':True, + 'output_mode':'scipy', + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + **kw, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # fetch dataframe + src_df = bundle[self.needs[0]] + if isinstance(src_df, (str, Path)): + df = self.load_path(src_df) + else: + df = src_df.copy() + + # fetch vocabulary (could be namespaced or generic) + vocab = bundle[self.needs[1]] + + # prepare dataframe + target = self.call_settings["target_column"] + df = df.dropna(subset=[target]) + + # build doc-word matrix + beaver = Beaver(**self.init_settings) + + cb = dict(self.call_settings) + if SAVE_DIR_BUNDLE_KEY in bundle: + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(parents=True, exist_ok=True) + cb.setdefault("save_path", out_dir) + + cb["options"]["vocabulary"] = vocab + X, _ = beaver.documents_words(dataset=df, **cb) + X = X.T.tocsr() + # store matrix under this block’s namespace + bundle[f"{self.tag}.{self.provides[0]}"] = X diff --git a/TELF/pipeline/blocks/beaver_participation_tensor_block.py b/TELF/pipeline/blocks/beaver_participation_tensor_block.py new file mode 100644 index 00000000..342e715f --- /dev/null +++ b/TELF/pipeline/blocks/beaver_participation_tensor_block.py @@ -0,0 +1,103 @@ +# blocks/beaver_participation_tensor_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import pandas as pd +from ...pre_processing.Beaver import Beaver +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class BeaverParticipationTensorBlock(AnimalBlock): + """ + Build an **(authors × papers × time)** “participation” tensor with *Beaver*. + + The underlying call is ``beaver.participation_tensor`` which yields + + * **X** – 3-mode tensor (author × paper × year) + * **authors** – list / array of author identifiers + * **paper_ids** – list / array of paper / EID identifiers + * **years** – temporal index (e.g. publication years) + + ───────────────────────────────────────────────────────────── + needs : ('df',) + provides : ('X', 'authors', 'paper_ids', 'years') + tag : 'BeaverPart' + """ + CANONICAL_NEEDS = ("df",) + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + target_columns: Tuple[str, str, str] = ("author_ids", "eid", "year"), + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("X", "authors", "paper_ids", "years"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "BeaverPart", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + # Beaver(...) constructor defaults + default_init = { + "n_jobs": -1, + "n_nodes": 1, + } + + # Arguments for beaver.participation_tensor(...) + default_call = { + "target_columns": target_columns, + "dimension_order": [0, 1, 2], # author, paper, year + "split_authors_with": ";", + "verbose": False, + "n_jobs": 1, + "n_nodes": 1, + # "save_path": injected dynamically if bundle['result_path'] exists + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # 1️⃣ Fetch DataFrame ------------------------------------------------ + src_df = bundle[self.needs[0]] + df = self.load_path(src_df) if isinstance(src_df, (str, Path)) else src_df.copy() + + # 2️⃣ Remove rows lacking any required column ----------------------- + for col in self.call_settings["target_columns"]: + if col in df.columns: + df = df.dropna(subset=[col]) + + # 3️⃣ Compose call-time settings ------------------------------------ + cb: Dict[str, Any] = dict(self.call_settings) + if "save_path" not in cb and SAVE_DIR_BUNDLE_KEY in bundle: + cb["save_path"] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + # 4️⃣ Execute Beaver ------------------------------------------------- + beaver = Beaver(**self.init_settings) + X, authors, paper_ids, years = beaver.participation_tensor(dataset=df, **cb) + + # 5️⃣ Store outputs under namespace --------------------------------- + ns = self.tag + bundle[f"{ns}.{self.provides[0]}"] = X + bundle[f"{ns}.{self.provides[1]}"] = authors + bundle[f"{ns}.{self.provides[2]}"] = paper_ids + bundle[f"{ns}.{self.provides[3]}"] = years diff --git a/TELF/pipeline/blocks/beaver_something_words_block.py b/TELF/pipeline/blocks/beaver_something_words_block.py new file mode 100644 index 00000000..f31d3e1e --- /dev/null +++ b/TELF/pipeline/blocks/beaver_something_words_block.py @@ -0,0 +1,107 @@ +# blocks/beaver_something_words_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +from ...pre_processing.Beaver import Beaver +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class BeaverSomethingWordsBlock(AnimalBlock): + """ + Build a sparse **something-words** matrix with *Beaver*. + + ───────────────────────────────────────────────────────────── + needs : ('df', 'vocabulary') + provides : ('X', 'keys', 'vocabulary') – matrix, ids, new vocab + tag : 'BeaverSW' + """ + CANONICAL_NEEDS = ("df", "vocabulary") + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + target_columns: tuple[str, str] = ("author_ids", "clean_abstract"), + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("X", "keys", "vocabulary"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "BeaverSW", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + # Settings forwarded to Beaver(...) + default_init = { + "n_jobs": -1, + "n_nodes": 1, + } + + # Settings forwarded to beaver.something_words(...) + default_call = { + "target_columns": target_columns, + "options": {"min_df": 5, "max_df": 0.5}, # vocabulary injected at run-time + "split_something_with": ";", + "matrix_type": "tfidf", + "highlighting": [], + "weights": [], + "verbose": False, + "return_object": True, + "output_mode": "scipy", + # "save_path" will be added automatically if bundle["result_path"] exists + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # 1️⃣ Fetch dataframe ------------------------------------------------ + src_df = bundle[self.needs[0]] + df = self.load_path(src_df) if isinstance(src_df, (str, Path)) else src_df.copy() + + # 2️⃣ Fetch vocabulary ---------------------------------------------- + vocab = bundle[self.needs[1]] + + # 3️⃣ Clean dataframe (drop rows with missing text columns) ---------- + for col in self.call_settings["target_columns"]: + if col in df.columns: + df = df.dropna(subset=[col]) + + # 4️⃣ Prepare Beaver & call spec ------------------------------------ + beaver = Beaver(**self.init_settings) + + cb: Dict[str, Any] = dict(self.call_settings) + + # inject vocabulary + cb.setdefault("options", {}) + cb["options"]["vocabulary"] = vocab + + # dynamic save path + if "save_path" not in cb and SAVE_DIR_BUNDLE_KEY in bundle: + cb["save_path"] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + # build the matrix + X, keys, vocab_new = beaver.something_words(dataset=df, **cb) + + # 5️⃣ Store under namespaced keys ------------------------------------ + ns = self.tag + bundle[f"{ns}.{self.provides[0]}"] = X # 'X' + bundle[f"{ns}.{self.provides[1]}"] = keys # 'keys' (e.g., author_ids) + bundle[f"{ns}.{self.provides[2]}"] = vocab_new # updated vocabulary diff --git a/TELF/pipeline/blocks/beaver_something_words_time_block.py b/TELF/pipeline/blocks/beaver_something_words_time_block.py new file mode 100644 index 00000000..075589d5 --- /dev/null +++ b/TELF/pipeline/blocks/beaver_something_words_time_block.py @@ -0,0 +1,108 @@ +# blocks/beaver_something_words_time_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import pandas as pd +from ...pre_processing.Beaver import Beaver +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class BeaverSomethingWordsTimeBlock(AnimalBlock): + """ + Build a **(something × words × time)** tensor with *Beaver*. + + The underlying call is ``beaver.something_words_time`` which returns: + + * **X** – 3-mode tensor (authors × words × time) + * **keys** – sequence of “something” IDs (e.g.\ author_ids) + * **vocabulary** – updated vocabulary list / dict + * **time_index** – sequence of temporal labels (e.g.\ years) + + ───────────────────────────────────────────────────────────── + needs : ('df', 'vocabulary') + provides : ('X', 'keys', 'vocabulary', 'time_index') + tag : 'BeaverSWT' + """ + CANONICAL_NEEDS = ("df", "vocabulary") + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + target_columns: Tuple[str, str, str] = ("author_ids", "clean_abstract", "year"), + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("X", "keys", "vocabulary", "time_index"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "BeaverSWT", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + # Settings passed to Beaver(...) + default_init = { + "n_jobs": -1, + "n_nodes": 1, + } + + # Settings passed to beaver.something_words_time(...) + default_call = { + "target_columns": target_columns, + # vocabulary injected at runtime + "tfidf_transformer": True, + "unfold_at": 1, + "verbose": False, + # "save_path" set automatically if bundle["result_path"] exists + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # 1️⃣ Load DataFrame ------------------------------------------------- + src_df = bundle[self.needs[0]] + df = self.load_path(src_df) if isinstance(src_df, (str, Path)) else src_df.copy() + + # 2️⃣ Obtain vocabulary -------------------------------------------- + vocab = bundle[self.needs[1]] + + # 3️⃣ Drop rows missing any target column --------------------------- + for col in self.call_settings["target_columns"]: + if col in df.columns: + df = df.dropna(subset=[col]) + + # 4️⃣ Build call-time settings -------------------------------------- + cb: Dict[str, Any] = dict(self.call_settings) + cb["vocabulary"] = vocab + + # save_path → result_path/BeaverSWT if none supplied + if "save_path" not in cb and SAVE_DIR_BUNDLE_KEY in bundle: + cb["save_path"] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + # 5️⃣ Execute Beaver ------------------------------------------------- + beaver = Beaver(**self.init_settings) + X, keys, vocab_new, time_axis = beaver.something_words_time(dataset=df, **cb) + + # 6️⃣ Store outputs -------------------------------------------------- + ns = self.tag + bundle[f"{ns}.{self.provides[0]}"] = X + bundle[f"{ns}.{self.provides[1]}"] = keys + bundle[f"{ns}.{self.provides[2]}"] = vocab_new + bundle[f"{ns}.{self.provides[3]}"] = time_axis diff --git a/TELF/pipeline/blocks/beaver_vocab_block.py b/TELF/pipeline/blocks/beaver_vocab_block.py new file mode 100644 index 00000000..68e72d5a --- /dev/null +++ b/TELF/pipeline/blocks/beaver_vocab_block.py @@ -0,0 +1,87 @@ +# blocks/beaver_vocab_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...pre_processing.Beaver import Beaver + + +class BeaverVocabBlock(AnimalBlock): + """ + Build a TF-IDF vocabulary from a DataFrame column. + + ───────────────────────────────────────────────────────────── + needs : ('df',) – the *latest* df in the bundle + provides : ('vocabulary',) – a list/dict of tokens + tag : 'BeaverVocab' – namespace for its outputs + """ + CANONICAL_NEEDS = ("df",) + + # NOTE: we have *no* conditional needs for this block. The argument is + # left in place so future versions can add them without changing callers. + def __init__( + self, + *, + col: str = "clean_title_abstract", + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("vocabulary",), + conditional_needs: Sequence[tuple[str, Any]] = (), + tag: str = "BeaverVocab", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + **kw, + ) -> None: + + self.col = col + default_init = { + "n_jobs": -1, + "n_nodes": 1, + } + + default_call = { + "target_column": self.col, + "max_df": 0.8, + "min_df": 10, + "max_features": 10_000, + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, # currently empty + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + **kw, + ) + + # ------------------------------------------------------------------ # + # main work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # fetch the DataFrame (generic or namespaced – resolved by DataBundle) + src_df = bundle[self.needs[0]] + if isinstance(src_df, (str, Path)): + df = self.load_path(src_df) + else: + df = src_df.copy() + + # clean up + target = self.call_settings["target_column"] + df = df.dropna(subset=[target]) + + # generate vocabulary + beaver = Beaver(**self.init_settings) + + # choose a save-to path if caller did not override it + call_cfg = dict(self.call_settings) + if "save_path" not in call_cfg and SAVE_DIR_BUNDLE_KEY in bundle: + call_cfg["save_path"] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + vocabulary = beaver.get_vocabulary(dataset=df, **call_cfg) + + # store under this block’s namespace + bundle[f"{self.tag}.{self.provides[0]}"] = vocabulary diff --git a/TELF/pipeline/blocks/cheetah_filter_block.py b/TELF/pipeline/blocks/cheetah_filter_block.py new file mode 100644 index 00000000..2b665712 --- /dev/null +++ b/TELF/pipeline/blocks/cheetah_filter_block.py @@ -0,0 +1,203 @@ +from pathlib import Path +from typing import Dict, Any, Optional +import json, warnings, pprint +import pandas as pd + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...applications import Cheetah + + +class CheetahFilterBlock(AnimalBlock): + """ + Hop-aware Cheetah filter with ultra-verbose logging. + """ + + CANONICAL_NEEDS = ("df", "query") + + # ----------------------------------------------------- INIT + def __init__( + self, + *, + needs=CANONICAL_NEEDS, + provides=("df", "cheetah_table"), + cheetah_columns: Optional[Dict[str, str]] = None, + tag: str = "CheetahFilter", + init_settings: Optional[Dict[str, Any]] = None, + call_settings: Optional[Dict[str, Any]] = None, + **kw, + ): + default_columns = { + "title": "title_abstract", + "abstract": "clean_title_abstract", + "year": "year", + "author_ids": "author_ids", + } + self.cheetah_columns = cheetah_columns or default_columns + + default_init = {"verbose": False, "use_hops": False} + default_call = { + "in_title": True, + "in_abstract": True, + "and_search": False, + "ngram_window_size": None, + "ngram_ordered": False, + "do_results_table": True, + } + + super().__init__( + needs=needs, + provides=provides, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + **kw, + ) + + # ------------------------------------------------------ RUN + def run(self, bundle: DataBundle) -> None: + + # 1 ─ load inputs + + df = self.load_path(bundle[self.needs[0]]) + df.info() + query = self.load_path(bundle[self.needs[1]]) + + print(f"\n[{self.tag}] ====================================================") + print(f"[{self.tag}] df.shape = {df.shape}") + if isinstance(query, (list, tuple)): + print(f"[{self.tag}] query len = {len(query)}") + for i, q in enumerate(query, 1): + print(f"[{self.tag}] • q[{i:02d}] = {q!r}") + else: + print(f"[{self.tag}] query = {query!r}") + + if "eid" not in df.columns: + raise ValueError("DataFrame must contain an 'eid' column.") + + cheetah_kwargs = dict(self.call_settings, query=query) + + # 2 ─ hop split + use_hops = self.init_settings["use_hops"] + if use_hops and "type" in df.columns: + max_t = df.type.max() + df_prev = df[df.type < max_t] + df_curr = df[df.type == max_t].reset_index(drop=True) + print(f"[{self.tag}] hop split → prev {df_prev.shape}, curr {df_curr.shape}") + else: + df_prev = df.iloc[0:0] + df_curr = df.reset_index(drop=True) + print(f"[{self.tag}] no hop split → curr {df_curr.shape}") + + # 3 ─ year numeric + year_col = self.cheetah_columns["year"] + df_curr[year_col] = pd.to_numeric(df_curr[year_col], errors="coerce") + print(f"[{self.tag}] '{year_col}' numeric (NaNs={df_curr[year_col].isna().sum()})") + + # 4 ─ text columns + title_col, abstract_col = self.cheetah_columns["title"], self.cheetah_columns["abstract"] + if title_col not in df_curr.columns: + df_curr[title_col] = ( + df_curr.get("title", pd.Series("", index=df_curr.index)).fillna("") + " " + + df_curr.get("abstract", pd.Series("", index=df_curr.index)).fillna("") + ) + print(f"[{self.tag}] built '{title_col}' by merging title+abstract") + for col in (title_col, abstract_col): + df_curr[col] = df_curr[col].fillna("").astype(str) + print(f"[{self.tag}] text columns ready") + + # 5 ─ output dir & index + root = Path(bundle.get(SAVE_DIR_BUNDLE_KEY, ".")) + out = root / self.tag + out.mkdir(parents=True, exist_ok=True) + idx_file = out / f"{self.tag}_index.p" + print(f"[{self.tag}] output dir = {out}") + + cheetah = Cheetah(verbose=self.init_settings["verbose"]) + cheetah.index( + df_curr, + columns={k: v for k, v in self.cheetah_columns.items() if k in Cheetah.COLUMNS}, + index_file=str(idx_file), + reindex=True, + ) + print(f"[{self.tag}] indexed rows = {len(df_curr)} → {idx_file.name}") + + # 5b ─ token hit map + def _token_hits(ch_obj, qlist): + print(f"[{self.tag}] ─ token hit map ─────────────────────────────") + items = qlist if isinstance(qlist, (list, tuple)) else [qlist] + for qi, q in enumerate(items, 1): + key = next(iter(q)) if isinstance(q, dict) else q + for tok in key.split(): + t = tok.lower() + hit_t = len(ch_obj.title_index.get(t, [])) + hit_a = len(ch_obj.abstract_index.get(t, [])) + print(f"[{self.tag}] q{qi:02d}:{t!r:<20} " + f"title={hit_t:4d} abstract={hit_a:4d}") + print(f"[{self.tag}] ─────────────────────────────────────────────") + _token_hits(cheetah, query) + + # 6 ─ window size + num_tok = len(str(query).split()) + if (ws := cheetah_kwargs.get("ngram_window_size")) is None or ws > num_tok: + cheetah_kwargs["ngram_window_size"] = num_tok + print(f"[{self.tag}] cheetah_kwargs:") + pprint.pp(cheetah_kwargs, compact=True, width=100) + + # 7 ─ search + try: + cheetah_df, cheetah_table = cheetah.search(**cheetah_kwargs) + except AssertionError as e: + warnings.warn(f"{self.tag}: n-gram assertion – {e}") + cheetah_df = df_curr.iloc[0:0].copy() + cheetah_table = pd.DataFrame(columns=["query"]) + print(f"[{self.tag}] search rows = {len(cheetah_df)}") + + if cheetah_table is not None and "filter_type" in cheetah_table.columns: + for _, r in cheetah_table.iterrows(): + if r["filter_type"] == "query": + print(f"[{self.tag}] query='{r['filter_value']}' hits={r['num_papers']}") + + # 7b ─ explain table enrich + if cheetah_table is not None and "included_ids" in cheetah_table.columns: + ids_series = cheetah_table["included_ids"].fillna("").astype(str) + cheetah_table["included_pos"] = ids_series.str.split(";").map( + lambda L: [int(x) for x in L if x.isdigit()] + ) + print(f"[{self.tag}] cheetah_table rows={len(cheetah_table)}") + + # 8 ─ merge & dedupe + merged = ( + pd.concat([df_prev, cheetah_df], ignore_index=True) + .drop_duplicates(subset=["eid"], keep="last") + .reset_index(drop=True) + ) + print(f"[{self.tag}] merge result = {merged.shape}") + + # 9 ─ refill columns + orig = df.drop_duplicates(subset=["eid"], keep="first").set_index("eid") + for col in orig.columns: + merged[col] = merged[col].fillna(merged["eid"].map(orig[col].to_dict())) + print(f"[{self.tag}] columns refilled") + + # 10 ─ save artefacts + merged_path = out / f"{self.tag}.csv" + merged.to_csv(merged_path, index=False, encoding="utf-8-sig") + self.register_checkpoint(self.provides[0], merged_path) + bundle[f"{self.tag}.{self.provides[0]}"] = merged + print(f"[{self.tag}] saved df → {merged_path}") + + if cheetah_table is not None: + table_path = out / "cheetah_table.csv" + cheetah_table.to_csv(table_path, index=False, encoding="utf-8-sig") + self.register_checkpoint(self.provides[1], table_path) + bundle[f"{self.tag}.{self.provides[1]}"] = cheetah_table + print(f"[{self.tag}] saved table → {table_path}") + else: + bundle[f"{self.tag}.{self.provides[1]}"] = None + + # save query as JSON + query_path = out / "query.json" + with open(query_path, "w", encoding="utf-8") as f: + json.dump(query, f, ensure_ascii=False, indent=2) + print(f"[{self.tag}] saved query → {query_path}") diff --git a/TELF/pipeline/blocks/clean_affiliations_block.py b/TELF/pipeline/blocks/clean_affiliations_block.py new file mode 100644 index 00000000..af24b62e --- /dev/null +++ b/TELF/pipeline/blocks/clean_affiliations_block.py @@ -0,0 +1,58 @@ +# blocks/clean_affiliations_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import pandas as pd +from ...helpers.filters import clean_affiliations + +from .base_block import AnimalBlock +from .data_bundle import DataBundle + + +class CleanAffiliationsBlock(AnimalBlock): + """ + Standardise and de-duplicate author affiliation strings. + + needs : ('df',) + (or any DataFrame key you choose) + provides : ('df',) + tag : 'CleanAffiliations' + """ + + CANONICAL_NEEDS = ("df",) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("df",), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "CleanAffiliations", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge({"verbose": True}, init_settings), + call_settings=self._merge({}, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + src = bundle[self.needs[0]] + df = self.load_path(src) if isinstance(src, (str, Path)) else src.copy() + + fixed_df = clean_affiliations(df, **self.call_settings) + + bundle[f"{self.tag}.{self.provides[0]}"] = fixed_df diff --git a/TELF/pipeline/blocks/clean_duplicates_block.py b/TELF/pipeline/blocks/clean_duplicates_block.py new file mode 100644 index 00000000..148eac84 --- /dev/null +++ b/TELF/pipeline/blocks/clean_duplicates_block.py @@ -0,0 +1,57 @@ +# blocks/clean_duplicates_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import pandas as pd +from ...helpers.frames import clean_duplicates + +from .base_block import AnimalBlock +from .data_bundle import DataBundle + + +class CleanDuplicatesBlock(AnimalBlock): + """ + Remove duplicate rows from a Scopus DataFrame. + + needs : ('df',) + provides : ('df',) + tag : 'CleanDuplicates' + """ + + CANONICAL_NEEDS = ("df",) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("df",), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "CleanDuplicates", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge({"verbose": True}, init_settings), + call_settings=self._merge({}, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + src = bundle[self.needs[0]] + df = self.load_path(src) if isinstance(src, (str, Path)) else src.copy() + + cleaned_df = clean_duplicates(df, **self.call_settings) + + bundle[f"{self.tag}.{self.provides[0]}"] = cleaned_df diff --git a/TELF/pipeline/blocks/combine_dir_loop_df_block.py b/TELF/pipeline/blocks/combine_dir_loop_df_block.py new file mode 100644 index 00000000..9a98cc21 --- /dev/null +++ b/TELF/pipeline/blocks/combine_dir_loop_df_block.py @@ -0,0 +1,78 @@ +from __future__ import annotations +from pathlib import Path +from typing import Any, Dict, List, Sequence +import pandas as pd + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY, RESULTS_DEFAULT + +class ConcatenateDFBlock(AnimalBlock): + """ + Collect the latest *df* from every sub-bundle stored in ⟨results⟩ + and concatenate them row-wise, tagging each row with the + great-grandparent folder name (e.g. "c5"). + """ + + CANONICAL_NEEDS = (RESULTS_DEFAULT,) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("df",), + source_key: str = "df", # ← key to pull from each sub-bundle + checkpoint_keys: Sequence[str] = ("df",), + tag: str = "ConcatDF", + **kw, + ): + self.source_key = source_key + super().__init__( + needs=needs, + provides=provides, + tag=tag, + checkpoint_keys=checkpoint_keys, + **kw, + ) + + def run(self, bundle: DataBundle) -> None: + results_list = bundle[self.needs[0]] + dfs: List[pd.DataFrame] = [] + + for subdict in results_list: + obj = subdict.get(self.source_key) + if obj is None: + raise KeyError( + f"Sub-bundle missing key {self.source_key!r}; " + f"available: {list(subdict)}" + ) + + # ── extract great-grandparent directory name ── + try: + p = Path(obj) + # file → Orca → results → c5 ⇒ parents[3].name == "c5" + source_dir = p.parent.parent.parent.name + except Exception: + source_dir = None + + df = self.load_path(obj) # handles Path / CSV / NPZ / DF… + if not isinstance(df, pd.DataFrame): + raise TypeError(f"Expected DataFrame, got {type(df)}") + + # add the new column before concatenation + df_copy = df.copy() + df_copy["source_dir"] = source_dir + + dfs.append(df_copy) + + consolidated = pd.concat(dfs, ignore_index=True) + + # (optional) persist + checkpoint + if SAVE_DIR_BUNDLE_KEY in bundle: + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(parents=True, exist_ok=True) + csv_path = out_dir / "consolidated_df.csv" + consolidated.to_csv(csv_path, index=False) + self.register_checkpoint(self.provides[0], csv_path) + + # save to bundle (namespaced + generic alias) + bundle[f"{self.tag}.{self.provides[0]}"] = consolidated diff --git a/TELF/pipeline/blocks/core_register_block.py b/TELF/pipeline/blocks/core_register_block.py new file mode 100644 index 00000000..70b61f94 --- /dev/null +++ b/TELF/pipeline/blocks/core_register_block.py @@ -0,0 +1,40 @@ +# blocks/merge_scopus_s2_block.py +from __future__ import annotations +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +class CoreRegister(AnimalBlock): + CANONICAL_NEEDS = ('df', ) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("df",), + tag: str = "CoreDf", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + super().__init__( + needs=needs, + provides=provides, + tag=tag, + init_settings=self._merge({"verbose": True}, init_settings), + call_settings=self._merge({}, call_settings), + verbose=verbose, + **kwargs, + ) + + def run(self, bundle: DataBundle) -> None: + df = bundle[self.needs[0]] + if SAVE_DIR_BUNDLE_KEY in bundle: + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(exist_ok=True, parents=True) + final_csv = out_dir / "core.csv" + df.to_csv(final_csv, index=False, encoding="utf-8-sig") + self.register_checkpoint(self.provides[0], final_csv) + bundle[f"{self.tag}.{self.provides[0]}"] = df diff --git a/TELF/pipeline/blocks/data_bundle.py b/TELF/pipeline/blocks/data_bundle.py new file mode 100644 index 00000000..ff7a291a --- /dev/null +++ b/TELF/pipeline/blocks/data_bundle.py @@ -0,0 +1,188 @@ +from collections import defaultdict +from typing import Any, Dict, Sequence, Optional, Iterator, Tuple + +SAVE_DIR_BUNDLE_KEY = 'save_path' +SOURCE_DIR_BUNDLE_KEY = 'dir' +DIR_LIST_BUNDLE_KEY = 'directories' +RESULTS_DEFAULT = 'results' + +class DataBundle: + """ + Stores values under base_keys, each with tagged versions and a '_latest' pointer. + Now supports: + - bundle.tag → NamespaceView for that tag + - bundle['tag'] → NamespaceView for that tag + - bundle.tag.key → same as bundle['tag']['key'] + """ + def __init__(self, initial: Optional[Dict[str, Any]] = None): + super().__setattr__('_store', defaultdict(dict)) + if initial: + for k, v in initial.items(): + if "." in k: + tag, base = k.split(".", 1) + else: + tag, base = "Init", k + self.__setitem__(base, v, tag=tag) + + def tags(self) -> set[str]: + """Return all tags currently in the bundle.""" + tags = set() + for bucket in self._store.values(): + tags.update(t for t in bucket.keys() if t != '_latest') + return tags + + def keys_by_tag(self, tag: str) -> list[str]: + """Return all base-keys that have a value under `tag`.""" + return [base for base, bucket in self._store.items() if tag in bucket] + + def print_tags_and_keys(self) -> None: + """ + Pretty-print each tag and the base-keys under it. + """ + for tag in sorted(self.tags()): + keys = self.keys_by_tag(tag) + print(f"{tag!r}: {keys}") + + def __getitem__(self, key: str) -> Any: + # If they ask for a bare tag, return the NamespaceView + if "." not in key and key in self.tags(): + return NamespaceView(self, key) + + # Otherwise fall back to normal behavior + tag, base = self._split(key) + bucket = self._store.get(base, {}) + if tag is None: + tag = bucket.get('_latest') + if tag is None: + raise KeyError(f"No value for key {base!r}") + if tag not in bucket: + raise KeyError(f"No {base!r} produced by tag {tag!r}") + return bucket[tag] + + def __setitem__(self, key: str, value: Any, *, tag: Optional[str] = None) -> None: + assumed = tag or self.__class__.__name__ + tag, base = self._split(key, assume_tag=assumed) + bucket = self._store.setdefault(base, {}) + bucket[tag] = value + bucket['_latest'] = tag + + def __getattr__(self, name: str) -> Any: + # If they access a tag by dot, return NamespaceView + if name in self.tags(): + return NamespaceView(self, name) + # If they access a base key by dot, return its latest value + if name in self._store: + return self.get(name) + raise AttributeError(f"{self.__class__.__name__!r} has no attribute {name!r}") + + def get(self, key: str, default: Any = None) -> Any: + tag, base = self._split(key) + bucket = self._store.get(base) + if not bucket: + return default + if tag is None: + tag = bucket.get('_latest') + if tag is None: + return default + return bucket.get(tag, default) + + def as_dict(self) -> Dict[str, Any]: + return {base: bucket[bucket['_latest']] for base, bucket in self._store.items()} + + def namespaced(self, tag: str) -> "NamespaceView": + return NamespaceView(self, tag) + + def __contains__(self, key: str) -> bool: + try: + _ = self[key] + return True + except KeyError: + return False + + def keys(self) -> Sequence[str]: + return list(self._store.keys()) + + def items(self) -> Iterator[Tuple[str, Any]]: + for k in self._store: + yield k, self.get(k) + + def values(self) -> Iterator[Any]: + for k in self._store: + yield self.get(k) + + def __iter__(self): + return iter(self._store) + + def __len__(self) -> int: + return len(self._store) + + def __repr__(self): + latest = {base: bucket.get('_latest') for base, bucket in self._store.items()} + return f"DataBundle(latest={latest})" + + def _split(self, key: str, assume_tag: Optional[str] = None) -> Tuple[Optional[str], str]: + if "." in key: + tag, base = key.split(".", 1) + else: + tag, base = None, key + return tag or assume_tag, base + + def __delitem__(self, key: str) -> None: + tag, base = self._split(key) + if base not in self._store: + raise KeyError(f"No such key: {key!r}") + bucket = self._store[base] + if tag is None: + tag = bucket.get("_latest") + if tag is None: + raise KeyError(f"No latest tag for base key: {base!r}") + if tag not in bucket: + raise KeyError(f"Tag {tag!r} not found for base key {base!r}") + del bucket[tag] + if len(bucket) <= 1: + del self._store[base] + else: + if bucket.get("_latest") == tag: + remaining = [k for k in bucket if k != "_latest"] + bucket["_latest"] = remaining[-1] if remaining else None + + def pop(self, key: str, default: Any = None) -> Any: + try: + value = self[key] + del self[key] + return value + except KeyError: + if default is not None: + return default + raise + +class NamespaceView: + """ + Read-only view for a single tag. + • view['foo'] → bundle['Tag.foo'] + • view.foo → bundle['Tag.foo'] + """ + def __init__(self, bundle: DataBundle, tag: str): + self._bundle = bundle + self._tag = tag + + def __getitem__(self, base: str): + return self._bundle[f"{self._tag}.{base}"] + + def __getattr__(self, name: str): + # allow dot access + try: + return self[name] + except KeyError as e: + raise AttributeError(f"{self.__class__.__name__!r} has no key {name!r}") from e + + def __contains__(self, base: str): + try: + _ = self._bundle[f"{self._tag}.{base}"] + return True + except KeyError: + return False + + def __repr__(self): + keys = self._bundle.keys_by_tag(self._tag) + return f"NamespaceView(tag={self._tag!r}, keys={keys})" diff --git a/TELF/pipeline/blocks/dataframe_load_block.py b/TELF/pipeline/blocks/dataframe_load_block.py new file mode 100644 index 00000000..20878967 --- /dev/null +++ b/TELF/pipeline/blocks/dataframe_load_block.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Sequence, Tuple, Union +import re +import pandas as pd + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SOURCE_DIR_BUNDLE_KEY, RESULTS_DEFAULT + +class LoadDfBlock(AnimalBlock): + """ + Load CSV file(s) under /, or directly from a full path. + + - If `full_path` is set, ignores the bundle and path_extension and loads from that path. + - Else if `recursive=True` or `regex` is provided, searches under the subfolder, + filters by regex, and returns either a single DataFrame or a list of file paths. + - Otherwise, loads exactly the file at /. + """ + CANONICAL_NEEDS = (SOURCE_DIR_BUNDLE_KEY,) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("df", "df_paths"), + path_extension: Path | str = Path(RESULTS_DEFAULT) / "papers.csv", + recursive: bool = False, + regex: str | None = None, + return_multiple: bool = False, + full_path: Path | str | None = None, + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "LoadDF", + verbose: bool = True, + **kw: Any, + ) -> None: + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + verbose=verbose, + **kw, + ) + self.path_extension = Path(path_extension) + self.recursive = recursive + self.regex = regex + self._pattern = re.compile(regex) if regex else None + self.return_multiple = return_multiple + self.full_path = Path(full_path) if full_path is not None else None + + def run(self, bundle: DataBundle) -> None: + # Determine base search location + if self.full_path: + search_root = self.full_path + else: + base_dir = Path(bundle[self.needs[0]]) + search_root = base_dir / self.path_extension + + # Collect CSV paths + if search_root.is_file(): + matches = [search_root] + elif (self.recursive or self.regex) and search_root.is_dir(): + globber = search_root.rglob if self.recursive else search_root.glob + candidates = globber("*.csv") + matches = [p for p in candidates if not self._pattern or self._pattern.search(p.name)] + elif not (self.recursive or self.regex) and search_root.exists(): + matches = [search_root] + else: + # No matches or wrong type + if self.full_path: + raise FileNotFoundError(f"No CSV files found at specified full_path {search_root!r}") + raise FileNotFoundError( + f"No CSV files matched under {search_root!r}" + + (f" with regex={self.regex!r}" if self.regex else "") + ) + + # Deduplicate and sort + matches = sorted(set(matches)) + + # Load or return paths + if len(matches) == 1 or not self.return_multiple: + df = pd.read_csv(matches[0]) + bundle[f"{self.tag}.df"] = df + bundle[f"{self.tag}.df_paths"] = [] + else: + bundle[f"{self.tag}.df"] = None + bundle[f"{self.tag}.df_paths"] = matches diff --git a/TELF/pipeline/blocks/field_filter_block.py b/TELF/pipeline/blocks/field_filter_block.py new file mode 100644 index 00000000..083af5c6 --- /dev/null +++ b/TELF/pipeline/blocks/field_filter_block.py @@ -0,0 +1,103 @@ +# blocks/filter_by_count_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple, Optional + +import pandas as pd +from .base_block import AnimalBlock +from .data_bundle import DataBundle + + +# ------------------------------------------------------------------ # +# helper # +# ------------------------------------------------------------------ # +def filter_by_count( + df: pd.DataFrame, + col: str = "author_ids", + min_count: Optional[int] = None, + max_count: Optional[int] = 30, + sep: Optional[str] = None, +) -> pd.DataFrame: + """See full docstring in original question.""" + sep = sep or ";" + + def _count(cell: Any) -> int: + if pd.isna(cell): + return 0 + if isinstance(cell, (list, tuple, set)): + return len(cell) + if isinstance(cell, str): + return len([p for p in cell.split(sep) if p]) + return 1 # anything else + + counts = df[col].apply(_count) + mask = pd.Series(True, index=df.index) + if min_count is not None: + mask &= counts >= min_count + if max_count is not None: + mask &= counts <= max_count + return df.loc[mask].copy() + + +# ------------------------------------------------------------------ # +# block # +# ------------------------------------------------------------------ # +class FilterByCountBlock(AnimalBlock): + """ + Filter a DataFrame by counting items in a column. + + needs : ('df',) + provides : ('df',) + tag : 'FilterByCount' + conditional : none (for now) + """ + CANONICAL_NEEDS = ('df', ) + + def __init__( + self, + *, + col: str = "author_ids", + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("df",), + conditional_needs: Sequence[Tuple[str, Any]] = (), # empty + tag: str = "FilterByCount", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + self.col = col + + default_init = {"verbose": True} + default_call = { + "col": self.col, + "min_count": None, + "max_count": 30, + "sep": None, + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # 1️⃣ pull the DataFrame (generic or namespaced) + src = bundle[self.needs[0]] + df = self.load_path(src) if isinstance(src, (str, Path)) else src.copy() + + # 2️⃣ filter + filtered_df = filter_by_count(df, **self.call_settings) + + # 3️⃣ store under this block’s namespace + bundle[f"{self.tag}.{self.provides[0]}"] = filtered_df diff --git a/TELF/pipeline/blocks/fox_block.py b/TELF/pipeline/blocks/fox_block.py new file mode 100644 index 00000000..d85f153b --- /dev/null +++ b/TELF/pipeline/blocks/fox_block.py @@ -0,0 +1,79 @@ +from pathlib import Path +from typing import Dict, Sequence, Any +import os +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...post_processing import Fox + +class FoxBlock(AnimalBlock): + """ + Post-processing + + ───────────────────────────────────────────────────────────── + always needs : ("df", "vocabulary", "model_path", "k", ), + provides : ('df',) + tag : 'Fox' + """ + CANONICAL_NEEDS = ("df", "vocabulary", "model_path", "k", ), + + def __init__( + self, + API_KEY:str=None, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("df", ), + make_summaries:bool=True, + conditional_needs: Sequence[tuple[str, Any]] = (), + tag: str = "Fox", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + **kw, + ): + + self.API_KEY=API_KEY + self.tag=tag + self.make_summaries = make_summaries + default_init = { + "summary_model":"gpt-3.5-turbo-instruct", + "verbose":False, + "debug":True, + } + default_call = {} + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + **kw, + ) + + + def run(self, bundle: DataBundle) -> None: + df = self.load_path(bundle[self.needs[0]]) + vocabulary = self.load_path(bundle[self.needs[1]]) + model_path = self.load_path(bundle[self.needs[2]]) + k = self.load_path(bundle[self.needs[3]]) + + if SAVE_DIR_BUNDLE_KEY in bundle: + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(exist_ok=True, parents=True) + else: + out_dir = self.tag + + fox = Fox(**self.init_settings) + + post_processed_df_path = fox.post_process( + npz_path=os.path.join(model_path, f"WH_k={k}.npz"), + vocabulary=vocabulary, + data=df, + output_dir=out_dir + ) + + if self.make_summaries: + fox.setApiKey(self.API_KEY) + fox.makeSummariesAndLabelsOpenAi( + processing_path=post_processed_df_path + ) + + bundle[f"{self.tag}.{self.provides[0]}"] = post_processed_df_path diff --git a/TELF/pipeline/blocks/function_block.py b/TELF/pipeline/blocks/function_block.py new file mode 100644 index 00000000..6743fa54 --- /dev/null +++ b/TELF/pipeline/blocks/function_block.py @@ -0,0 +1,80 @@ +# blocks/function_block.py +from __future__ import annotations + +from typing import Any, Callable, Dict, Tuple, Sequence + +from .data_bundle import DataBundle +from .base_block import AnimalBlock + + +class FunctionBlock(AnimalBlock): + """ + Wrap **any** Python callable so it becomes a pipeline block. + + The callable is invoked as + + result = function_call(*[bundle[n] for n in needs], **call_settings) + + and its return value(s) are written back to the bundle under this block’s + namespace (`tag.provides[i]`). + + Because the class derives from *AnimalBlock* it automatically supports: + + * **tagging** – no two blocks clobber each other’s outputs. + * **conditional_needs** – declare runtime-dependent inputs if you ever need + them (none by default). + """ + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + needs: Tuple[str, ...], + provides: Tuple[str, ...], + function_call: Callable[..., Any], + call_settings: Dict[str, Any] | None = None, + tag: str = "Function", + conditional_needs: Sequence[Tuple[str, Any]] = (), # optional + **kw, + ) -> None: + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + call_settings=call_settings or {}, + **kw, + ) + + self.function_call = function_call + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # build positional argument list from declared needs + args = [bundle[key] for key in self.needs] + + # kwargs to pass through + kwargs = ( + self.call_settings + if isinstance(self.call_settings, dict) + else vars(self.call_settings) + ) + + # call the user function + result = self.function_call(*args, **kwargs) + + # write back under this block’s namespace + if len(self.provides) == 1: + bundle[f"{self.tag}.{self.provides[0]}"] = result + else: + if not isinstance(result, (list, tuple)) or len(result) != len(self.provides): + raise ValueError( + f"{self.__class__.__name__} expected {len(self.provides)} return " + f"values but got {type(result)} with length {getattr(result, '__len__', lambda: '?')()}" + ) + for key, val in zip(self.provides, result): + bundle[f"{self.tag}.{key}"] = val diff --git a/TELF/pipeline/blocks/group_summary_block.py b/TELF/pipeline/blocks/group_summary_block.py new file mode 100644 index 00000000..b0f449c4 --- /dev/null +++ b/TELF/pipeline/blocks/group_summary_block.py @@ -0,0 +1,323 @@ +from __future__ import annotations +from pathlib import Path +from typing import Any, Dict, Sequence, List + +import pandas as pd +import matplotlib.pyplot as plt + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from .pipeline_summary_block import PipelineSummaryBlock + + +def _bundle_items(db: DataBundle): + """ + Iterate over a DataBundle as (pseudo-key, value) pairs so the collectors can + treat it like a regular mapping. + + This yields: + - "Tag.docs_summary_df" for docs_summary_df entries + - "Tag.df" for any *.df entries + - other values are propagated for recursion + + Parameters + ---------- + db : DataBundle + The data bundle to iterate. + + Yields + ------ + tuple[str, Any] + Pseudo-key and corresponding value. + """ + for base, bucket in db._store.items(): # noqa: SLF001 + for tag, val in bucket.items(): + if tag == "_latest": + continue + yield f"{tag}.{base}", val + + +class GroupSummaryBlock(AnimalBlock): + """ + Aggregate document counts across any mixture of RepeatLoopBlock and + DirectoryLoopBlock nests. Falls back to counting every `*.df` when no + PipelineSummaryBlock `docs_summary_df` objects are present. + + Parameters + ---------- + summary_block_tag : str, optional + Tag used by PipelineSummaryBlock to generate summary dataframes, + by default "PipelineSummary" + call_settings : dict[str, Any] | None, optional + Settings for plotting and ordering, by default None + needs : sequence[str] | None, optional + Keys required from the bundle, by default None (uses SAVE_DIR_BUNDLE_KEY) + provides : sequence[str], optional + Keys provided to the bundle after run, by default + ("group_summary_df", "group_summary_plot") + tag : str, optional + Tag for this block, by default "GroupSummary" + """ + + def __init__( + self, + *, + summary_block_tag: str = "PipelineSummary", + call_settings: Dict[str, Any] | None = None, + needs: Sequence[str] | None = None, + provides: Sequence[str] = ("group_summary_df", "group_summary_plot"), + tag: str = "GroupSummary", + **kw: Any, + ): + default_call = { + "log_scale": True, + "plot_kwargs": {}, + "label_bars": True, + "order_by_pipeline": False, + "desired_order": [] + } + if needs is None: + needs = (SAVE_DIR_BUNDLE_KEY,) + super().__init__( + needs=tuple(needs), + provides=provides, + tag=tag, + call_settings=self._merge(default_call, call_settings), + **kw + ) + self.summary_tag = summary_block_tag + + def _is_summary_df(self, key: str) -> bool: + """ + Determine if a key corresponds to a PipelineSummaryBlock dataframe. + + Parameters + ---------- + key : str + The bundle key to check. + + Returns + ------- + bool + True if it matches docs_summary_df pattern. + """ + return ( + key == "docs_summary_df" + or (key.endswith(".docs_summary_df") and key.startswith(self.summary_tag)) + ) + + def _ctx_label(self, ctx: Dict[str, str]) -> str: + """ + Construct a group label from directory and iteration context. + + Parameters + ---------- + ctx : dict[str, str] + Context dictionary containing 'dir' and/or 'iter'. + + Returns + ------- + str + Combined label, e.g. "dir/iter" or single value. + """ + if "dir" in ctx and "iter" in ctx: + return f"{ctx['dir']}/{ctx['iter']}" + return ctx.get("dir") or ctx.get("iter") or "all" + + def _collect_summaries( + self, obj: Any, ctx: Dict[str, str], out: List[pd.DataFrame] + ) -> None: + """ + Recursively collect PipelineSummaryBlock dataframes and tag with group. + + Parameters + ---------- + obj : Any + Nested object (dict, list, DataBundle). + ctx : dict[str, str] + Current directory/iteration context. + out : list[pandas.DataFrame] + Collected dataframes with added 'group' column. + """ + if isinstance(obj, dict): + new_ctx = ctx.copy() + if "dir" in obj and obj["dir"] is not None: + new_ctx["dir"] = Path(obj["dir"]).name + if "iter" in obj and obj["iter"] is not None: + new_ctx["iter"] = f"iter_{int(obj['iter'])}" + for k, v in obj.items(): + if self._is_summary_df(k) and isinstance(v, pd.DataFrame): + df = v.copy() + df["group"] = self._ctx_label(new_ctx) + out.append(df) + if isinstance(v, (dict, list, DataBundle)): + self._collect_summaries(v, new_ctx, out) + + elif isinstance(obj, list): + for item in obj: + self._collect_summaries(item, ctx, out) + + elif isinstance(obj, DataBundle): + for k, v in _bundle_items(obj): + if self._is_summary_df(k) and isinstance(v, pd.DataFrame): + df = v.copy() + df["group"] = self._ctx_label(ctx) + out.append(df) + if isinstance(v, (dict, list, DataBundle)): + self._collect_summaries(v, ctx, out) + + def _collect_raw_dfs( + self, obj: Any, ctx: Dict[str, str], out: List[pd.DataFrame] + ) -> None: + """ + Recursively collect any DataFrame (*.df) and count rows as fallback. + + Parameters + ---------- + obj : Any + Nested object to search. + ctx : dict[str, str] + Current context. + out : list[pandas.DataFrame] + Collected dataframes with block_tag, num_docs, group. + """ + if isinstance(obj, dict): + new_ctx = ctx.copy() + if "dir" in obj and obj["dir"] is not None: + new_ctx["dir"] = Path(obj["dir"]).name + if "iter" in obj and obj["iter"] is not None: + new_ctx["iter"] = f"iter_{int(obj['iter'])}" + for k, v in obj.items(): + if k.endswith(".df") and isinstance(v, pd.DataFrame): + block_tag = k.split(".")[0] + out.append( + pd.DataFrame({ + "block_tag": [block_tag], + "num_docs": [len(v)], + "group": [self._ctx_label(new_ctx)], + }) + ) + if isinstance(v, (dict, list, DataBundle)): + self._collect_raw_dfs(v, new_ctx, out) + + elif isinstance(obj, list): + for item in obj: + self._collect_raw_dfs(item, ctx, out) + + elif isinstance(obj, DataBundle): + for k, v in _bundle_items(obj): + if k.endswith(".df") and isinstance(v, pd.DataFrame): + block_tag = k.split(".")[0] + out.append( + pd.DataFrame({ + "block_tag": [block_tag], + "num_docs": [len(v)], + "group": [self._ctx_label(ctx)], + }) + ) + if isinstance(v, (dict, list, DataBundle)): + self._collect_raw_dfs(v, ctx, out) + + def run(self, bundle: DataBundle) -> DataBundle: + """ + Execute the block: collect summaries or raw dfs, pivot, reorder columns, + and generate/save both CSV and bar plot with optional labels. + + Parameters + ---------- + bundle : DataBundle + The input data bundle containing dfs and summary dfs. + + Returns + ------- + DataBundle + The same bundle with added: + - "GroupSummary.group_summary_df" (pandas.DataFrame) + - "GroupSummary.group_summary_plot" (Path to PNG) + + Raises + ------ + ValueError + If no DataFrames are found to summarise. + """ + # Ensure at least one PipelineSummary exists + if not any(self._is_summary_df(k) for k in bundle): + try: + PipelineSummaryBlock(tag=self.summary_tag)(bundle) + except Exception: + pass + + dfs: List[pd.DataFrame] = [] + self._collect_summaries(bundle, {}, dfs) + + if not dfs: + self._collect_raw_dfs(bundle, {}, dfs) + + if not dfs: + raise ValueError(f"{self.tag}: could not locate any DataFrames to summarise.") + + combined = pd.concat(dfs, ignore_index=True) + pivot = ( + combined + .pivot_table( + index="group", columns="block_tag", + values="num_docs", aggfunc="sum", fill_value=0 + ) + .apply(pd.to_numeric, errors="coerce").fillna(0).astype(int) + ) + + # Reorder columns if requested + if self.call_settings.get("pipeline_order", False): + pipeline_order = self.call_settings.get("pipeline_order", []) + if pipeline_order: + valid = [col for col in pipeline_order if col in pivot.columns] + pivot = pivot.reindex(columns=valid) + + # Save and plot + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(parents=True, exist_ok=True) + + csv_path = out_dir / "group_summary.csv" + pivot.to_csv(csv_path) + + png_path = out_dir / "group_summary.png" + if not pivot.empty and pivot.values.sum() > 0: + fig, ax = plt.subplots( + figsize=(max(6, 0.3 * pivot.shape[1]), + max(4, 0.5 * pivot.shape[0])) + ) + pivot.plot( + kind="bar", stacked=False, ax=ax, + **self.call_settings.get("plot_kwargs", {}) + ) + ax.set_xlabel("Group") + ax.set_ylabel("Documents") + ax.set_title("Documents per Block by Group") + # rotate and align x-tick labels + ax.tick_params(axis="x", labelrotation=45) + for lbl in ax.get_xticklabels(): + lbl.set_ha('right') + + if self.call_settings.get("label_bars", True): + for container in ax.containers: + for bar in container: + height = bar.get_height() + if height > 0: + ax.text( + bar.get_x() + bar.get_width() / 2, + height, str(height), + ha="center", va="bottom", fontsize=8, + rotation=90 # rotation angle + ) + + plt.tight_layout() + if self.call_settings['log_scale']: + plt.yscale("log") + plt.savefig(png_path, dpi=150) + plt.close() + + bundle[f"{self.tag}.group_summary_df"] = pivot + bundle[f"{self.tag}.group_summary_plot"] = png_path + self.register_checkpoint("group_summary_df", csv_path) + self.register_checkpoint("group_summary_plot", png_path) + return bundle diff --git a/TELF/pipeline/blocks/hnmfk_block.py b/TELF/pipeline/blocks/hnmfk_block.py new file mode 100644 index 00000000..313c8267 --- /dev/null +++ b/TELF/pipeline/blocks/hnmfk_block.py @@ -0,0 +1,131 @@ +# blocks/hnmfk_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...factorization import HNMFk +from copy import deepcopy + +class HNMFkBlock(AnimalBlock): + CANONICAL_NEEDS = ("X", ) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("hnmfk_model", "saved_path"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "HNMFk", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kw: Any, + ) -> None: + # 1) your single default nmfk_params dict: + base_nmfk = { + "n_perturbs": 5, + "n_iters": 500, + "epsilon": 0.015, + "n_jobs": -1, + "init": "nnsvd", + "use_gpu": False, + "save_output": True, + "collect_output": True, + "predict_k_method": "sill", + "verbose": True, + "nmf_verbose": False, + "transpose": False, + "sill_thresh": 0.8, + "pruned": True, + "nmf_method": "nmf_fro_mu", + "calculate_error": True, + "predict_k": True, + "use_consensus_stopping": 0, + "calculate_pac": True, + "consensus_mat": True, + "perturb_type": "uniform", + "perturb_multiprocessing": False, + "perturb_verbose": False, + "simple_plot": True, + "k_search_method": "bst_pre", + "H_sill_thresh": 0.1, + "clustering_method": "kmeans", + "device": -1, + } + + # 2) build a clean copy of default_init + default_init = { + "nmfk_params": [base_nmfk], + "cluster_on": "H", + "depth": 1, + "sample_thresh": 5, + "K2": False, + "Ks_deep_min": 1, + "Ks_deep_max": 20, + "Ks_deep_step": 1, + "random_identifiers": False, + "root_node_name": "Root", + } + + # 3) if the user passed nmfk_params in init_settings, special-case it: + user_init = deepcopy(init_settings) or {} + if "nmfk_params" in user_init: + overrides = user_init.pop("nmfk_params") + # make sure we have a list of dicts + if isinstance(overrides, dict): + overrides = [overrides] + # for each override dict, shallow-merge it onto a fresh base_nmfk + merged_list = [] + for ov in overrides: + merged = {**base_nmfk, **ov} + merged_list.append(merged) + default_init["nmfk_params"] = merged_list + + # 4) now do your normal (shallow) merge for everything else + def _merge(default: Dict[str, Any], override: Dict[str, Any] | None): + return {**default, **(override or {})} + + merged_init = _merge(default_init, user_init) + + # 5) build call_settings as before + default_call = { + "Ks": range(1, 21), + "from_checkpoint": True, + "save_checkpoint": True, + } + merged_call = _merge(default_call, call_settings) + + # 6) finally hand everything up to the base class + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=merged_init, + call_settings=merged_call, + verbose=verbose, + **kw, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # load matrix X + src = bundle[self.needs[0]] + X = self.load_path(src) if isinstance(src, (str, Path)) else src + + init_cfg = dict(self.init_settings) + if "experiment_name" not in init_cfg and SAVE_DIR_BUNDLE_KEY in bundle: + init_cfg["experiment_name"] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + # fit hierarchical model + model = HNMFk(**init_cfg) + model.fit(X, **self.call_settings) + + # store under namespace + bundle[f"{self.tag}.{self.provides[0]}"] = model + bundle[f"{self.tag}.{self.provides[1]}"] = model.experiment_save_path diff --git a/TELF/pipeline/blocks/lmf_block.py b/TELF/pipeline/blocks/lmf_block.py new file mode 100644 index 00000000..3a359a7d --- /dev/null +++ b/TELF/pipeline/blocks/lmf_block.py @@ -0,0 +1,122 @@ +from pathlib import Path +from typing import Dict, Sequence, Any +import pickle +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +from ...factorization.decompositions.lmf import LogisticMatrixFactorization + +class LMFBlock(AnimalBlock): + CANONICAL_NEEDS = ("X", "MASK",) + + def __init__( + self, + *, + threshold:float=0.5, + needs = CANONICAL_NEEDS, + provides = ("W", "H", "row_bias", "col_bias", "losses", "Xtilda", "Xtilda_bool", "model",), + conditional_needs: Sequence[tuple[str, Any]] = (), # none today + tag: str = "LMF", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + **kw, + ): + """ + A component for performing low-rank matrix factorization with masking. + + This module is designed to work within a pipeline framework where it declares + its input needs and what it provides after execution. + + Parameters: + ---------- + threshold : float, default=0.5 + A threshold value used during internal computations (e.g., for binary masking or evaluation). + needs : tuple of str, default=("X", "MASK",) + The names of inputs this module requires. + provides : tuple of str, default=("W", "H", "row_bias", "col_bias", "losses", "Xtilda", "Xtilda_bool", "model") + The names of outputs this module will produce and provide. + conditional_needs : Sequence[tuple[str, Any]], default=() + Additional inputs needed conditionally based on runtime logic (unused currently). + tag : str, default="LMF" + A label used to tag or identify the module in logs or output. + init_settings : dict, optional + Additional settings used during initialization. + call_settings : dict, optional + Settings to control behavior during the execution or "call" phase. + **kw : dict + Arbitrary additional keyword arguments. + + Needs: + ------ + - "X": Input data matrix. + - "MASK": A mask matrix indicating observed vs. missing entries. + + Provides: + --------- + - "W": Left factor matrix. + - "H": Right factor matrix. + - "row_bias": Bias values for rows. + - "col_bias": Bias values for columns. + - "losses": Loss values recorded during training. + - "Xtilda": Reconstructed matrix. + - "Xtilda_bool": Binary thresholded version of the reconstruction. + - "model": The trained model or relevant object representing it. + """ + self.threshold=threshold + default_init = { + "k": 5, + "l2_p": 1e-6, + "epochs": 3000, + "learning_rate": 0.001, + "tolerance": 1e-3, + "device": 2, + "random_state": 1 + } + default_call = { + "plot_loss":False + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + **kw, + ) + + + def run(self, bundle: DataBundle) -> None: + # 1 — load X + X = self.load_path(bundle[self.needs[0]]) + MASK = self.load_path(bundle[self.needs[1]]) + + if SAVE_DIR_BUNDLE_KEY in bundle: + path = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + else: + path = self.tag + + model = LogisticMatrixFactorization(**self.init_settings) + W, H, row_bias, col_bias, losses = model.fit(X, MASK, **self.call_settings) + Xtilda = model.predict(W, H, row_bias, col_bias) + Xtilda_bool = model.map_probabilities_to_binary(Xtilda, threshold=self.threshold) + + pickle.dump( + {"W":W, + "H":H, + "row_bias":row_bias, "col_bias":col_bias, + "losses":losses, + "Xtilda":Xtilda, + "Xtilda_bool":Xtilda_bool}, + open(path, "wb")) + + # 4 — store + bundle[f"{self.tag}.{self.provides[0]}"] = W + bundle[f"{self.tag}.{self.provides[1]}"] = H + bundle[f"{self.tag}.{self.provides[2]}"] = row_bias + bundle[f"{self.tag}.{self.provides[3]}"] = col_bias + bundle[f"{self.tag}.{self.provides[4]}"] = losses + bundle[f"{self.tag}.{self.provides[5]}"] = Xtilda + bundle[f"{self.tag}.{self.provides[6]}"] = Xtilda_bool + bundle[f"{self.tag}.{self.provides[7]}"] = model diff --git a/TELF/pipeline/blocks/load_terms_block.py b/TELF/pipeline/blocks/load_terms_block.py new file mode 100644 index 00000000..a11e0b17 --- /dev/null +++ b/TELF/pipeline/blocks/load_terms_block.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Union +from collections import defaultdict + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SOURCE_DIR_BUNDLE_KEY +from ...applications import CheetahTermFormatter +from ...helpers.terms import resolve_substitution_conflicts + +def find_markdown_file( + dir_path: Optional[Path], + *, + explicit: Optional[Path] = None, + preferred_name: str = "single_stage_terms.md", +) -> Path: + if explicit is None and dir_path is None: + raise FileNotFoundError("No markdown file specified and SOURCE_DIR is missing.") + + if explicit: + md = Path(explicit) + if not md.exists(): + raise FileNotFoundError(f"Explicit markdown path {md} does not exist.") + return md + + candidates = list(dir_path.glob("*.md")) + if not candidates: + raise FileNotFoundError(f"No markdown files found in {dir_path}") + + for c in candidates: + if c.name == preferred_name: + return c + return candidates[0] + +class LoadTermsBlock(AnimalBlock): + """ + Load a Markdown term list (or passed-in overrides) and place four + artefacts into the bundle: + + • .terms (raw terms list) + • .substitutions (forward map) + • .substitutions_reverse (reverse map) + • .query (Cheetah-ready query list) + + If drop_conflicts is True, mappings are conflict-resolved. + """ + + CANONICAL_NEEDS: Sequence[str] = () + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("terms", "substitutions", "substitutions_reverse", "query"), + tag: str = "Terms", + init_settings: Optional[Dict[str, Any]] = None, + call_settings: Optional[Dict[str, Any]] = None, + ): + default_call = { + SOURCE_DIR_BUNDLE_KEY: None, + "overrides": None, + "drop_conflicts": True, + } + merged_call = self._merge(default_call, call_settings) + + conds = [ + ( + SOURCE_DIR_BUNDLE_KEY, + lambda bundle, blk: blk.call_settings[SOURCE_DIR_BUNDLE_KEY] is None, + ) + ] + + super().__init__( + needs=needs, + provides=provides, + tag=tag, + conditional_needs=conds, + init_settings=self._merge({}, init_settings), + call_settings=merged_call, + ) + self.tag = tag + + def run(self, bundle: DataBundle) -> None: + overrides = self.call_settings["overrides"] + drop_conflicts = self.call_settings["drop_conflicts"] + + if overrides is not None: + sub_forward, sub_reverse, terms = overrides + else: + dir_path = ( + Path(bundle[SOURCE_DIR_BUNDLE_KEY]) + if SOURCE_DIR_BUNDLE_KEY in bundle + else None + ) + explicit = self.call_settings[SOURCE_DIR_BUNDLE_KEY] + md_file = find_markdown_file( + dir_path, + explicit=Path(explicit) if explicit else None, + ) + + fmt = CheetahTermFormatter( + markdown_file=md_file, + lower=True, + substitutions=True, + drop_conflicts=drop_conflicts, + ) + sub_forward, sub_reverse = fmt.get_substitution_maps() + terms = fmt.get_terms() + + if drop_conflicts: + clean_forward, dropped = resolve_substitution_conflicts(sub_forward, warn=True) + if dropped: + terms = self._prune_terms_list(terms, dropped) + else: + clean_forward = sub_forward + + # build reverse map + rev: Dict[str, List[str]] = defaultdict(list) + for src, tgt in clean_forward.items(): + rev[tgt].append(src) + clean_reverse = dict(rev) + + # stash raw terms + maps + bundle[f"{self.tag}.terms"] = terms + bundle[f"{self.tag}.substitutions"] = clean_forward + bundle[f"{self.tag}.substitutions_reverse"] = clean_reverse + + # build Cheetah-ready query + def _flatten(entry: Any) -> Union[str, Dict[str, List[str]]]: + if isinstance(entry, str): + return entry + if isinstance(entry, dict) and len(entry) == 1: + term, spec = next(iter(entry.items())) + # nested positives/negatives + if isinstance(spec, dict) and 'positives' in spec and 'negatives' in spec: + pos = [f"+{p}" for p in spec['positives']] + neg = spec['negatives'] + return {term: pos + neg} + # direct list of dependents + if isinstance(spec, list): + return {term: spec} + raise TypeError(f"Cannot normalize term entry: {entry!r}") + + query_list: List[Any] = [_flatten(e) for e in terms] + bundle[f"{self.tag}.query"] = query_list + + @staticmethod + def _prune_terms_list(terms: List[Any], dropped: set[str]) -> List[Any]: + pruned: List[Any] = [] + for entry in terms: + if isinstance(entry, str): + if entry not in dropped: + pruned.append(entry) + else: + kept = {k: v for k, v in entry.items() if k not in dropped} + if kept: + pruned.append(kept) + return pruned diff --git a/TELF/pipeline/blocks/merge_scopus_s2_block.py b/TELF/pipeline/blocks/merge_scopus_s2_block.py new file mode 100644 index 00000000..a1fd23dd --- /dev/null +++ b/TELF/pipeline/blocks/merge_scopus_s2_block.py @@ -0,0 +1,62 @@ +# blocks/merge_scopus_s2_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import pandas as pd +from ...helpers.frames import merge_scopus_s2 + +from .base_block import AnimalBlock +from .data_bundle import DataBundle + + +class MergeScopusS2Block(AnimalBlock): + """ + Merge a *Scopus* DataFrame with a *Semantic Scholar (S2)* DataFrame. + + needs : ('df', 'df') + (override if your bundle uses different keys) + provides : ('df',) + tag : 'MergeScopusS2' + """ + + # CANONICAL_NEEDS = ("Scopus.df", "S2.df") + + def __init__( + self, + *, + needs: Sequence[str] = ("Scopus.df", "S2.df"), + provides: Sequence[str] = ("df",), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "MergeScopusS2", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge({"verbose": True}, init_settings), + call_settings=self._merge({}, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + src_left = bundle[self.needs[0]] + src_right = bundle[self.needs[1]] + + left_df = self.load_path(src_left) if isinstance(src_left, (str, Path)) else src_left.copy() + right_df = self.load_path(src_right) if isinstance(src_right, (str, Path)) else src_right.copy() + + merged_df = merge_scopus_s2(left_df, right_df, **self.call_settings) + + + bundle[f"{self.tag}.{self.provides[0]}"] = merged_df diff --git a/TELF/pipeline/blocks/nmfk_block.py b/TELF/pipeline/blocks/nmfk_block.py new file mode 100644 index 00000000..38f45e89 --- /dev/null +++ b/TELF/pipeline/blocks/nmfk_block.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Dict, Sequence, Any, Tuple + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...factorization import NMFk + + +_NEEDS_MASK = {"nmf_recommender", "wnmf", "bnmf"} # methods that require a mask + + +class NMFkBlock(AnimalBlock): + """ + Wrapper around **NMFk**. + + always needs 'X' + needs 'MASK' only when init_settings['nmf_method'] ∈ {_NEEDS_MASK} + provides : ("results", "model_path", ) + """ + CANONICAL_NEEDS = ("X", ) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("nmfk_model", "nmfk_model_path" ), + tag: str = "NMFk", + conditional_needs: Sequence[Tuple[str, Any]] | None = None, + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + **kw, + ) -> None: + + default_init = { + # ---------- NMFk defaults ---------- + "n_perturbs": 5, + "n_iters": 500, + "epsilon": 0.015, + "n_jobs": -1, + "init": "nnsvd", + "use_gpu": False, + "save_output": True, + "collect_output": True, + "predict_k_method": "sill", + "verbose": True, + "nmf_verbose": False, + "transpose": False, + "sill_thresh": 0.8, + "pruned": True, + "nmf_method": "nmf_fro_mu", # ← drives MASK requirement + "calculate_error": True, + "predict_k": True, + "use_consensus_stopping": 0, + "calculate_pac": True, + "consensus_mat": True, + "perturb_type": "uniform", + "perturb_multiprocessing": False, + "perturb_verbose": False, + "simple_plot": True, + "k_search_method": "bst_pre", + "H_sill_thresh": 0.1, + "clustering_method": "kmeans", + "device": -1, + "nmf_obj_params": {}, + } + + default_call = { + "Ks": range(1, 21), + "name": "Example_NMFk", + "note": "This is an example run of NMFk", + } + + init_cfg = {**default_init, **(init_settings or {})} + + # ------------------------------------------------------------------ + # conditional MASK rule: *register it only if nmf_method needs one* + # ------------------------------------------------------------------ + conds = list(conditional_needs or []) + if init_cfg["nmf_method"] in _NEEDS_MASK: + # an always-true condition is enough because we add it ONLY + # in the branch that needs it + conds.append(("MASK", lambda _b, _s: True)) + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conds, + tag=tag, + init_settings=self._merge(init_cfg, init_settings), + call_settings=self._merge(default_call, call_settings), + **kw, + ) + + # ------------------------------------------------------------------ # + # run # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # 1 — load X + X_val = bundle[self.needs[0]] + X = self.load_path(X_val) if isinstance(X_val, (str, Path)) else X_val + + if self.init_settings["nmf_method"] in _NEEDS_MASK: + self.init_settings["nmf_obj_params"]["MASK"] = bundle["MASK"] + + init_cfg = dict(self.init_settings) + if SAVE_DIR_BUNDLE_KEY not in init_cfg and SAVE_DIR_BUNDLE_KEY in bundle: + init_cfg[SAVE_DIR_BUNDLE_KEY] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + model = NMFk(**init_cfg) + results = model.fit(X, **self.call_settings) + + # 4 — store + bundle[f"{self.tag}.{self.provides[0]}"] = results + bundle[f"{self.tag}.{self.provides[1]}"] = model.save_path_full + + def _after_checkpoint_skip(self, bundle: DataBundle) -> None: + """Re-insert fast-to-make steps after checkpoint reload.""" + # steps = self.__rebuild_steps(bundle) + # bundle[f"{self.tag}.{self.provides[1]}"] = steps \ No newline at end of file diff --git a/TELF/pipeline/blocks/orca_block.py b/TELF/pipeline/blocks/orca_block.py new file mode 100644 index 00000000..343ee3d7 --- /dev/null +++ b/TELF/pipeline/blocks/orca_block.py @@ -0,0 +1,83 @@ +from pathlib import Path +from typing import Any, Callable, Dict, List, Tuple, Optional, Sequence + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +from ...pre_processing import Orca +from ...pre_processing.Orca.utils import orca_summary +from ...helpers.frames import add_num_known_col, prep_affiliations, drop_columns_if_exist +from ...helpers.filters import clean_affiliations + +class OrcaBlock(AnimalBlock): + + CANONICAL_NEEDS = ('df', ) + + def __init__( + self, + needs = CANONICAL_NEEDS, + provides = ('df', 'map'), + tag = 'Orca', + conditional_needs: Sequence[Tuple[str, Any]] = (), + *, + init_settings: Dict[str, Any] = None, + call_settings: Dict[str, Any] = None, + **kw, + ) -> None: + + default_init = {'verbose':True} + default_call = {} + + super().__init__( + needs = needs, + provides = provides, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + conditional_needs=conditional_needs, + tag=tag, + **kw, + ) + + + def run(self, bundle: DataBundle) -> None: + raw = bundle[self.needs[0]] + if isinstance(raw, (str, Path)): + df = self.load_path(raw) + else: + df = raw.copy() + + df = drop_columns_if_exist(df, cols = ['slic_affiliations', 'slic_author_ids', 'slic_authors']) + + orca = Orca(**self.init_settings) + df = clean_affiliations(df) + + orca_map_df = orca.run(df) + df = clean_affiliations(df) + + orca_map_df = orca_map_df.dropna(subset=['scopus_ids']).reset_index(drop=True) + orca_map_df = add_num_known_col(orca_map_df) + + orca_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + orca_dir.mkdir(parents=True, exist_ok=True) + + + df = orca.apply(df) + df = clean_affiliations(df) + + df = prep_affiliations(df) + + if 'type' in df.columns: + orca_summary_path = save_path=orca_dir / 'type_summary.csv' + summary_df = orca_summary(df, save_path=orca_summary_path) + + orca_map_df_path = orca_dir / 'orca_map.csv' + orca_map_df.to_csv(orca_map_df_path, index=False) + + orca_df_path = orca_dir / 'orca_df.csv' + df.to_csv(orca_df_path, index=False) + + self.register_checkpoint(self.provides[0], orca_df_path) + self.register_checkpoint(self.provides[1], orca_map_df_path) + + bundle[f"{self.tag}.{self.provides[0]}"] = df + bundle[f"{self.tag}.{self.provides[1]}"] = orca_map_df diff --git a/TELF/pipeline/blocks/peacock_stats_block.py b/TELF/pipeline/blocks/peacock_stats_block.py new file mode 100644 index 00000000..8a290e0f --- /dev/null +++ b/TELF/pipeline/blocks/peacock_stats_block.py @@ -0,0 +1,239 @@ +# pipeline/blocks/peacock_stats_block.py +from __future__ import annotations +from pathlib import Path +from typing import Any, Dict, Sequence, Optional, Tuple + +import pandas as pd + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +# Peacock aggregation function +from ...post_processing.Peacock.Utility import aggregate_ostats + +# Peacock plotting functions (the ones you pasted from Plot/plot.py) +from ...post_processing.Peacock.Plot import ( + plot_heatmap, + plot_bar, + plot_scatter, +) + +# Convenience aliases +plot_hist = plot_bar +plot_scatter3D = plot_scatter + + +class PeacockStatsBlock(AnimalBlock): + CANONICAL_NEEDS: Tuple[str, ...] = ("df", SAVE_DIR_BUNDLE_KEY) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("outpath",), + hist_stats: Sequence[str] = ("paper_count", "num_citations"), + hist_ylabels: Optional[Dict[str, str]] = None, + col_names: Optional[Dict[str, str]] = None, + affiliation_palette: Optional[Dict[str, str]] = None, + country: Optional[str] = None, + **kw: Any, + ) -> None: + super().__init__( + needs=needs, + provides=provides, + tag="PeacockStats", + init_settings={}, + call_settings={}, + **kw, + ) + self.hist_stats = tuple(hist_stats) + self.hist_ylabels = hist_ylabels or { + "paper_count": "Number of Papers", + "num_citations": "Number of Citations", + "attribution_percentage": "Attribution Percentage", + } + self.col_names = col_names or { + "id": "eid", + "authors": "slic_authors", + "author_ids": "slic_author_ids", + "affiliations": "slic_affiliations", + "funding": "funding", + "citations": "num_citations", + "references": "references", + } + self.affiliation_palette = affiliation_palette or {} + self.country = country + + def run(self, bundle: DataBundle) -> None: + # 1) Inputs & cleanup + df: pd.DataFrame = bundle["df"] + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) + out_dir.mkdir(parents=True, exist_ok=True) + + # ensure affiliation column is string + aff_col = self.col_names["affiliations"] + df[aff_col] = df[aff_col].apply(lambda x: x if isinstance(x, str) else str(x)) + + df = ( + df + .dropna(subset=[ + self.col_names["id"], + self.col_names["authors"], + self.col_names["author_ids"], + aff_col, + ]) + .assign(year=lambda d: d.year.astype(int)) + ) + + filters = {"country": self.country} if self.country else None + + # 2) Write top‐100 CSVs + author_stats = aggregate_ostats(df, key="author_id", top_n=100, col_names=self.col_names, filters=filters, by_year=False) + affiliation_stats = aggregate_ostats(df, key="affiliation_id", top_n=100, col_names=self.col_names, filters=filters, by_year=False) + author_stats.to_csv(out_dir / "top_authors.csv", index=False) + affiliation_stats.to_csv(out_dir / "top_affiliations.csv", index=False) + + # 3) Shared “top‐10 by citations” args + auth_args = dict( + key="author_id", + top_n=10, + sort_by="num_citations", + col_names=self.col_names, + by_year=True, + filters=filters, + ) + aff_args = dict( + key="affiliation_id", + top_n=10, + sort_by="num_citations", + col_names=self.col_names, + by_year=True, + filters=filters, + ) + + # 4) Heatmaps — pivot then call plot_heatmap + # ─ Authors, citations + auth_heat = aggregate_ostats(df, **auth_args) + pivot_c = auth_heat.pivot(index="year", columns="author", values="num_citations").fillna(0) + plot_heatmap( + pivot_c, + cmap="jet", + interpolation="gaussian", + fname=str(out_dir / "author_heatmap_citations.png"), + interactive=False, + title="Author Citations by Year", + xlabel="Author", + ylabel="Year", + ) + # ─ Authors, papers + pivot_p = auth_heat.pivot(index="year", columns="author", values="paper_count").fillna(0) + plot_heatmap( + pivot_p, + cmap="jet", + interpolation="gaussian", + fname=str(out_dir / "author_heatmap_papers.png"), + interactive=False, + title="Author Papers by Year", + xlabel="Author", + ylabel="Year", + ) + + # ─ Affiliations, citations + aff_heat = aggregate_ostats(df, **aff_args) + pivot_c2 = aff_heat.pivot(index="year", columns="affiliation", values="num_citations").fillna(0) + plot_heatmap( + pivot_c2, + cmap="jet", + interpolation="gaussian", + fname=str(out_dir / "affiliation_heatmap_citations.png"), + interactive=False, + title="Affiliation Citations by Year", + xlabel="Affiliation", + ylabel="Year", + ) + # ─ Affiliations, papers + pivot_p2 = aff_heat.pivot(index="year", columns="affiliation", values="paper_count").fillna(0) + plot_heatmap( + pivot_p2, + cmap="jet", + interpolation="gaussian", + fname=str(out_dir / "affiliation_heatmap_papers.png"), + interactive=False, + title="Affiliation Papers by Year", + xlabel="Affiliation", + ylabel="Year", + ) + + # 5) Histograms (bar‐plots) + auth_hist = aggregate_ostats(df, **{**auth_args, "by_year": False}) + plot_hist( + auth_hist, + x="author", + ys=list(self.hist_stats), + fname=str(out_dir / "author_hist.png"), + interactive=False, + # cmap="husl", # ← remove this line... + title="Author Statistics Histogram", + xlabel="Author", + ylabel=self.hist_ylabels[self.hist_stats[0]], + ) + aff_hist = aggregate_ostats(df, **{**aff_args, "by_year": False}) + plot_hist( + aff_hist, + x="affiliation", + ys=list(self.hist_stats), + fname=str(out_dir / "affiliation_hist.png"), + interactive=False, + # cmap="husl", + title="Affiliation Statistics Histogram", + xlabel="Affiliation", + ylabel=self.hist_ylabels[self.hist_stats[0]], + ) + + # 6) 3-D scatter + plot_scatter3D( + df, + x="paper_count", + y="attribution_percentage", + z="num_citations", + agg_func=aggregate_ostats, + agg_kwargs=auth_args, + fname=str(out_dir / "author_scatter.png"), + interactive=False, + log_z=True, + hue="affiliation", + labels="author", + title="Author Stats Scatter3D", + xlabel="Paper Count", + ylabel="Attribution Percentage", + zlabel="Num. Citations", + base_palette=self.affiliation_palette, + ) + plot_scatter3D( + df, + x="paper_count", + y="attribution_percentage", + z="num_citations", + agg_func=aggregate_ostats, + agg_kwargs=aff_args, + fname=str(out_dir / "affiliation_scatter.png"), + interactive=False, + log_z=True, + hue="country", + labels="affiliation", + title="Affiliation Stats Scatter3D", + xlabel="Paper Count", + ylabel="Attribution Percentage", + zlabel="Num. Citations", + base_palette=self.affiliation_palette, + ) + + # 7) Dummy checkpoint under your single `provides` key + if SAVE_DIR_BUNDLE_KEY in bundle: + ckpt_dir = out_dir / self.tag + ckpt_dir.mkdir(parents=True, exist_ok=True) + final_csv = ckpt_dir / "none.csv" + final_csv.write_text("") # empty placeholder + self.register_checkpoint(self.provides[0], final_csv) + bundle[f"{self.tag}.{self.provides[0]}"] = final_csv + # (no return: AnimalBlock.__call__ returns the bundle) diff --git a/TELF/pipeline/blocks/pipeline_summary_block.py b/TELF/pipeline/blocks/pipeline_summary_block.py new file mode 100644 index 00000000..e56dbb05 --- /dev/null +++ b/TELF/pipeline/blocks/pipeline_summary_block.py @@ -0,0 +1,141 @@ +from __future__ import annotations +from pathlib import Path +from typing import Dict, Sequence, Any, Tuple, List +import json +import pandas as pd +import matplotlib.pyplot as plt +import numpy as np +import datetime as dt + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class PipelineSummaryBlock(AnimalBlock): + """ + Final pipeline block: scan every block directory, pick up each + block's 'df' checkpoint (if present), count its rows, and plot + them in execution order. + """ + + def __init__( + self, + *, + needs: Sequence[str] = (), + provides: Sequence[str] = ("docs_summary_df", "docs_summary_plot"), + tag: str = "PipelineSummary", + checkpoint_keys: Sequence[str] = ("docs_summary_df", "docs_summary_plot"), + **kw, + ): + super().__init__( + needs=needs, + provides=provides, + tag=tag, + checkpoint_keys=checkpoint_keys, + **kw, + ) + + def _discover(self, root: Path) -> List[Tuple[str, Path, float]]: + """ + Return a list of (block_tag, df_file, mtime) for each sub-directory + that has a __checkpoints__.json with a 'df' entry. + """ + found: List[Tuple[str, Path, float]] = [] + + for blk_dir in root.iterdir(): + if not blk_dir.is_dir(): + continue + tag = blk_dir.name + ckpt = blk_dir / "__checkpoints__.json" + if not ckpt.is_file(): + continue + + try: + mapping = json.loads(ckpt.read_text(encoding="utf-8")) + except Exception: + continue + + df_path_str = mapping.get("df") + if not df_path_str: + continue + + fp = Path(df_path_str) + if not fp.is_file(): + continue + + mtime = fp.stat().st_mtime + found.append((tag, fp, mtime)) + + found.sort(key=lambda tup: tup[2]) + return found + + def run(self, bundle: DataBundle) -> None: + if SAVE_DIR_BUNDLE_KEY not in bundle: + raise KeyError( + f"{self.tag}: bundle lacks {SAVE_DIR_BUNDLE_KEY}; cannot discover checkpoint folders." + ) + + root = Path(bundle[SAVE_DIR_BUNDLE_KEY]) + triples = self._discover(root) + + rows: List[Tuple[str, int, str]] = [] + for tag, fp, mtime in triples: + try: + obj = self.load_path(fp) + count = obj.shape[0] if hasattr(obj, "shape") else len(obj) + except Exception: + count = -1 + rows.append(( + tag, + count, + dt.datetime.fromtimestamp(mtime).isoformat(sep=" ", timespec="seconds") + )) + + summary_df = pd.DataFrame( + rows, + columns=["block_tag", "num_docs", "first_write_time"] + ) + + # save CSV + out_dir = root / self.tag + out_dir.mkdir(parents=True, exist_ok=True) + csv_path = out_dir / "docs_per_block.csv" + img_path = out_dir / "docs_per_block.png" + summary_df.to_csv(csv_path, index=False) + + # plotting with log scale, individual colors, and annotations + n_bars = len(summary_df) + cmap = plt.get_cmap("tab20") + colors = cmap(np.linspace(0, 1, n_bars)) + + plt.figure(figsize=(max(4, 0.5 * n_bars), 4)) + bars = plt.bar( + summary_df["block_tag"], + summary_df["num_docs"], + color=colors, + ) + plt.yscale("log") + + for bar, count in zip(bars, summary_df["num_docs"]): + height = bar.get_height() + plt.text( + bar.get_x() + bar.get_width() / 2, + height, + f"{count}", + ha="center", + va="bottom", + fontsize=8, + ) + + plt.xticks(rotation=45, ha="right") + plt.ylabel("Documents") + plt.title("Documents per Block (execution order)") + plt.tight_layout() + plt.savefig(img_path, dpi=150) + plt.close() + + # bundle values + checkpoints + bundle[f"{self.tag}.docs_summary_df"] = summary_df + bundle[f"{self.tag}.docs_summary_plot"] = img_path + self.register_checkpoint("docs_summary_df", csv_path) + self.register_checkpoint("docs_summary_plot", img_path) diff --git a/TELF/pipeline/blocks/post_process_cluster_analysis_block.py b/TELF/pipeline/blocks/post_process_cluster_analysis_block.py new file mode 100644 index 00000000..0fd13002 --- /dev/null +++ b/TELF/pipeline/blocks/post_process_cluster_analysis_block.py @@ -0,0 +1,158 @@ +from __future__ import annotations +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import numpy as np +import pandas as pd + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...post_processing.ArcticFox import ClusteringAnalyzer + + +class ClusteringAnalyzerBlock(AnimalBlock): + """ + Wraps the ClusteringAnalyzer with conditional inputs based on a single `mode` parameter. + + always needs : ('df',) + conditional needs : + - flat : 'W','H','vocab' + - hierarchical: 'hnmfk_model','vocab' + - label : none (uses `cluster_col` from call_settings) + - passthrough : none + provides : ('result',) → path or list of paths to generated CSV(s) + tag : 'ClusteringAnalyzer' + """ + CANONICAL_NEEDS: Tuple[str, ...] = ("df",) + VALID_MODES = ["nmf", "hnmf", "label", None] + + def __init__( + self, + *, + mode: str | None = None, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("clusters_path",), + tag: str = "ClusteringAnalyzer", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + **kw: Any, + ) -> None: + if mode not in self.VALID_MODES: + raise ValueError(f"Invalid mode '{mode}'. Valid modes: {self.VALID_MODES}") + self.mode = mode + + # shared init settings for ClusteringAnalyzer + default_init = { + "top_n_words": 50, + "out_dir": None, + "archive_subdir": "archive", + "default_clean_col": "clean_title_abstract", + "wordcloud_size": (800, 800), + "max_font_size": 80, + "contour_width": 1, + "col_year": "year", + "col_type": "type", + "col_cluster": "cluster", + "col_cluster_coords": "cluster_coordinates", + "col_similarity": "similarity_to_cluster_centroid", + "table_filename": "table_H-clustering.csv", + "cluster_doc_map_filename": "cluster_documents_map.txt", + "top_words_filename": "top_words.csv", + "probs_filename": "probabilities_top_words.csv", + "clusters_info_filename": "clusters_information.csv", + "documents_info_filename": "documents_information.csv", + } + default_call = { + "cluster_col": None, + "clean_cols_name": None, + "process_parents": True, + "skip_completed": True, + } + + init_cfg = {**default_init, **(init_settings or {})} + call_cfg = {**default_call, **(call_settings or {})} + + # build conditional_needs based on mode + conds: list[Tuple[str, Any]] = [] + if self.mode and self.mode == self.VALID_MODES[0]: + conds = [ + ("nmfk_model", lambda b, s: True), + ("nmfk_model_path", lambda b, s: True), + ("vocabulary", lambda b, s: True), + ] + elif self.mode and self.mode == self.VALID_MODES[1]: + conds = [ + ("hnmfk_model", lambda b, s: True), + ("vocabulary", lambda b, s: True), + ] + # 'label' and 'passthrough' modes require no extra bundle keys; + # label-based logic uses call_settings['cluster_col'] at runtime + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conds, + tag=tag, + init_settings=init_cfg, + call_settings=call_cfg, + **kw, + ) + + def run(self, bundle: DataBundle) -> None: + # 1) load the input DataFrame + df_val = bundle[self.needs[0]] + df = self.load_path(df_val) if isinstance(df_val, (str, Path)) else df_val + + # 2) instantiate the analyzer + analyzer = ClusteringAnalyzer(**self.init_settings) + result = None + + # dispatch based on mode + if not self.mode: + if not self.init_settings.get("out_dir"): + analyzer.out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + result = analyzer.analyze(df) + + elif self.mode == self.VALID_MODES[1]: + + h_model = bundle["hnmfk_model"] + vocab = bundle["vocabulary"] + + if not self.init_settings.get("out_dir"): + analyzer.out_dir = str(Path(h_model.experiment_save_path)) + + result = analyzer.analyze( + df, + hnmfk_model = h_model, + vocab = vocab, + clean_cols_name = self.call_settings["clean_cols_name"], + process_parents = self.call_settings["process_parents"], + skip_completed = self.call_settings["skip_completed"], + ) + elif self.mode == self.VALID_MODES[0] : + nmfk_model = bundle['nmfk_model'] + nmfk_model_path = bundle['nmfk_model_path'] + + + if not self.init_settings.get("out_dir"): + analyzer.out_dir = str(Path(nmfk_model_path).parent) + + W, H = nmfk_model['W'], nmfk_model['H'] + + vocab = bundle["vocabulary"] + result = analyzer.analyze( + df, + W = W, + H = H, + vocab = vocab, + ) + elif self.mode == self.VALID_MODES[2] : + if not self.init_settings.get("out_dir"): + analyzer.out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + result = analyzer.analyze( + df, + cluster_col = self.call_settings["cluster_col"], + ) + + # 4) register the output + bundle[f"{self.tag}.{self.provides[0]}"] = result diff --git a/TELF/pipeline/blocks/post_process_label_analysis_block.py b/TELF/pipeline/blocks/post_process_label_analysis_block.py new file mode 100644 index 00000000..e6f17529 --- /dev/null +++ b/TELF/pipeline/blocks/post_process_label_analysis_block.py @@ -0,0 +1,176 @@ +# pipeline/blocks/label_analyzer_block.py +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Dict, Iterable, Sequence, Tuple, Union + +import pandas as pd + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...post_processing.ArcticFox.label_analyzer import LabelAnalyzer + + +# ───────────────────────────────────────────────────────────────────────────── +# helper – harvest all “cluster_for_k=*.csv” under a root folder +def _collect_csvs(root: Path) -> Iterable[Path]: + pat = re.compile(r"cluster_for_k=\d+\.csv") + for p in root.rglob("*.csv"): + if pat.fullmatch(p.name): + yield p.resolve() + + +class LabelAnalyzerBlock(AnimalBlock): + """ + Generate concise topic labels using `LabelAnalyzer` for: + + • a single DataFrame / CSV + • a list / tuple of CSV paths (e.g., HNMF-k leaf outputs) + • an entire directory tree containing `cluster_for_k=*.csv` + + provides → ('result',) : dict[csv_path → dict[int,str]] + and : list[str] of the written labels.csv files + """ + + CANONICAL_NEEDS: Tuple[str, ...] = ("clusters_path",) + + # --------------------------------------------------------------------- # + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("result","label_paths"), + tag: str = "LabelAnalyzer", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + **kw: Any, + ) -> None: + + # ---------------- default configs -------------------------------- # + default_init = dict( + embedding_model="SCINCL", + distance_metric="cosine", + text_cols=["title", "abstract"], + tfidf_top_k=12, + include_bigrams=True, + ) + default_call = dict( + provider="ollama", + model_name=None, + openai_api_key=None, # can also be supplied in the bundle + cluster_strategy=None, # auto-detect per-file + cluster_col="cluster", + top_n_words=20, + num_candidates=12, + use_gpu=False, + ) + + self.init_settings = {**default_init, **(init_settings or {})} + self.call_settings = {**default_call, **(call_settings or {})} + + # conditional need for the OpenAI key + conds = [] + if self.call_settings["provider"].lower() == "openai": + conds = [("openai_api_key", lambda b, s: "openai_api_key" in b)] + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conds, + tag=tag, + init_settings=self.init_settings, + call_settings=self.call_settings, + **kw, + ) + + # --------------------------------------------------------------------- # + def _label_single_csv(self, csv_path: Path, labeler: LabelAnalyzer) -> Dict[int, str]: + df = pd.read_csv(csv_path) + + # choose cluster strategy automatically + strat = ("column" if self.call_settings["cluster_col"] in df.columns + else "single") if self.call_settings["cluster_strategy"] is None \ + else self.call_settings["cluster_strategy"] + + labels = labeler.label_texts( + df, + provider = self.call_settings["provider"], + model_name = self.call_settings["model_name"], + openai_api_key = self.call_settings["openai_api_key"], + cluster_strategy = strat, + cluster_col = self.call_settings["cluster_col"], + top_n_words = self.call_settings["top_n_words"], + num_candidates = self.call_settings["num_candidates"], + use_gpu = self.call_settings["use_gpu"], + ) + + # write labels.csv next to input + out_csv = csv_path.with_name("labels.csv") + pd.DataFrame(sorted(labels.items()), columns=["cluster", "label"]).to_csv(out_csv, index=False) + return labels + + # --------------------------------------------------------------------- # + def run(self, bundle: DataBundle) -> None: # imperative – no docstring lint + src = bundle[self.needs[0]] + + # resolve inputs to an iterable of CSV files or a single DataFrame + csv_paths: list[Path] = [] + dataframe_param: Union[pd.DataFrame, None] = None + + if isinstance(src, pd.DataFrame): + dataframe_param = src + elif isinstance(src, (str, Path)): + p = Path(src) + if p.is_dir(): + csv_paths = list(_collect_csvs(p)) + elif p.is_file(): + csv_paths = [p] + else: + raise FileNotFoundError(p) + elif isinstance(src, (list, tuple)): + csv_paths = [Path(s) for s in src] + else: + raise TypeError(f"Unsupported df input type {type(src)}") + + # out-dir fallback for temp files (single-DF case) + base_out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + base_out_dir.mkdir(parents=True, exist_ok=True) + + labeler = LabelAnalyzer(**self.init_settings) + result_dict: Dict[str, Dict[int, str]] = {} + written_csvs: list[str] = [] + + # ---------------------------------- many CSVs --------------------- # + if csv_paths: + for csv_p in csv_paths: + labels = self._label_single_csv(csv_p, labeler) + result_dict[str(csv_p)] = labels + written_csvs.append(str(csv_p.with_name("labels.csv"))) + + # ---------------------------------- single DataFrame -------------- # + if dataframe_param is not None: + # auto cluster strategy + strat = ("column" if self.call_settings["cluster_col"] in dataframe_param.columns + else "single") if self.call_settings["cluster_strategy"] is None \ + else self.call_settings["cluster_strategy"] + + labels = labeler.label_texts( + dataframe_param, + provider = self.call_settings["provider"], + model_name = self.call_settings["model_name"], + openai_api_key = self.call_settings["openai_api_key"], + cluster_strategy = strat, + cluster_col = self.call_settings["cluster_col"], + top_n_words = self.call_settings["top_n_words"], + num_candidates = self.call_settings["num_candidates"], + use_gpu = self.call_settings["use_gpu"], + ) + out_csv = base_out_dir / "labels.csv" + pd.DataFrame(sorted(labels.items()), columns=["cluster", "label"]).to_csv(out_csv, index=False) + result_dict[""] = labels + written_csvs.append(str(out_csv)) + + # register outputs + bundle[f"{self.tag}.{self.provides[0]}"] = result_dict # python dict + bundle[f"{self.tag}.{self.provides[1]}"] = written_csvs # list of paths diff --git a/TELF/pipeline/blocks/rescalk_block.py b/TELF/pipeline/blocks/rescalk_block.py new file mode 100644 index 00000000..6d10ecd2 --- /dev/null +++ b/TELF/pipeline/blocks/rescalk_block.py @@ -0,0 +1,118 @@ +# blocks/rescalc_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +from ...factorization import RESCALk +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +class RESCALkBlock(AnimalBlock): + """ + Pipeline wrapper around **RESCALk** (tensor factorisation). + + needs + ----- + ('X',) – a 3-mode tensor (NumPy ndarray, sparse COO, or on-disk path) + + provides + -------- + ('results', 'model_path') + + * **results** – output of :py:meth:`RESCALk.fit` + * **model_path** – directory where the model artefacts were saved + + tag + --- + 'RESCALk' + """ + + CANONICAL_NEEDS = ("X",) + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("results", "model_path"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "RESCALk", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + # ------------------- default constructor params ------------------- # + default_init = { + "n_perturbs": 12, + "n_iters": 50, + "epsilon": 0.015, + "n_jobs": -1, + "n_nodes": 1, + "init": "nnsvd", + "use_gpu": True, + # save_path will be injected automatically if not supplied + "save_output": True, + "verbose": True, + "pruned": False, + "rescal_verbose": False, + "calculate_error": True, + "rescal_func": None, + "rescal_obj_params": {}, + "simple_plot": True, + "rescal_method": "rescal_fro_mu", + "get_plot_data": True, + "perturb_type": "uniform", + "perturb_multiprocessing": False, + "perturb_verbose": False, + } + + # ------------------- default arguments to `.fit()` ---------------- # + default_call = { + "Ks": range(1, 7), # 1 … 6 + "name": "RESCALk", + "note": "This is an example run of RESCALk", + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # 1️⃣ Load tensor X -------------------------------------------------- + src_X = bundle[self.needs[0]] + X = self.load_path(src_X) if isinstance(src_X, (str, Path)) else src_X + + # 2️⃣ Ensure `save_path` exists / is injected ----------------------- + init_cfg: Dict[str, Any] = dict(self.init_settings) + if SAVE_DIR_BUNDLE_KEY not in init_cfg and SAVE_DIR_BUNDLE_KEY in bundle: + init_cfg[SAVE_DIR_BUNDLE_KEY] = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + # 3️⃣ Fit RESCALk ---------------------------------------------------- + model = RESCALk(**init_cfg) + results = model.fit( + X, + self.call_settings["Ks"], + self.call_settings["name"], + self.call_settings["note"], + ) + + # 4️⃣ Store outputs in the bundle ----------------------------------- + ns = self.tag + bundle[f"{ns}.{self.provides[0]}"] = results + bundle[f"{ns}.{self.provides[1]}"] = model.save_path_full diff --git a/TELF/pipeline/blocks/s2_block.py b/TELF/pipeline/blocks/s2_block.py new file mode 100644 index 00000000..cd69fda6 --- /dev/null +++ b/TELF/pipeline/blocks/s2_block.py @@ -0,0 +1,148 @@ +from pathlib import Path +from typing import Dict, Sequence, Any +import pandas as pd +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...pre_processing import SemanticScholar +from ...applications.Penguin import Penguin + +from ...helpers.file_system import copy_all_files + +class S2Block(AnimalBlock): + CANONICAL_NEEDS = ('df', ) + + def __init__( + self, + use_penguin: bool = False, + penguin_settings: Dict[str, Any] = { + "uri": "localhost:27017", + "db_name": "Penguin", + "username": None, + "password": None, + }, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("df",), + tag: str = "S2", + *, + init_settings: Dict[str, Any] = None, + call_settings: Dict[str, Any] = None, + **kw, + ) -> None: + self.use_penguin = use_penguin + self.penguin_settings = penguin_settings + + default_init = { + 'key': None, # S2 API key + 'mode': 'fs', # filesystem caching + 'name': None, # real cache dir if provided + 'ignore': None, + 'verbose': True, + } + super().__init__( + needs=needs, + provides=provides, + init_settings=self._merge(default_init, init_settings), + call_settings={}, + tag=tag, + **kw, + ) + + + def run(self, bundle: DataBundle) -> None: + # 1) Load input & extract DOIs + raw = bundle[self.needs[0]] + df_in = self.load_path(raw) if isinstance(raw, (str, Path)) else raw.copy() + requested = df_in.doi.dropna().astype(str).tolist() + if self.verbose: + print(f"[{self.tag}] candidate DOIs count: {len(requested)}") + + # 2) Prepare output dir & CSV + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(parents=True, exist_ok=True) + csv_path = out_dir / "s2_df.csv" + cache_dir = out_dir / "S2_CACHE" + cache_dir.mkdir(exist_ok=True, parents=True) + if self.verbose: + print(f"[{self.tag}] cache dir: {cache_dir}") + + # 3) What’s already in Penguin? + if self.use_penguin: + penguin = Penguin(**self.penguin_settings) + seen_db = set(penguin.distinct_dois(dois=requested)) + else: + seen_db = set() + if self.verbose: + print(f"[{self.tag}] seen in Mongo: {len(seen_db)}") + + # 4) What’s already on disk? + if csv_path.exists(): + df_csv = pd.read_csv(csv_path) + seen_local = set(df_csv.doi.dropna().astype(str).tolist()) + else: + df_csv = pd.DataFrame() + seen_local = set() + if self.verbose: + print(f"[{self.tag}] seen locally: {len(seen_local)}") + + # 5) Figure out DOIs to fetch + to_fetch = [d for d in requested if d not in seen_local and d not in seen_db] + if self.verbose: + print(f"[{self.tag}] to fetch count: {len(to_fetch)}") + + # 6) Fetch missing via SemanticScholar + df_new = pd.DataFrame(columns=df_csv.columns) + if to_fetch: + settings = dict(self.init_settings, name=str(cache_dir)) + s2 = SemanticScholar(**settings) + df_api, s2_targets = s2.search(to_fetch, mode="paper", n=0) + if self.verbose: + print(f"[{self.tag}] fetched via API: {len(df_api)}") + + # 6a) upsert into Penguin + if self.use_penguin and not df_api.empty: + penguin.add_many_documents(cache_dir, source="S2", overwrite=False) + if self.verbose: + print(f"[{self.tag}] bulk-upserted JSON files into Mongo") + + # 6b) re-pull only those we just added + # collect all S2 IDs from the API response + df_api = penguin.id_search( + ids=[f"s2id:{uid}" for uid in set(s2_targets)], + as_pandas=True + ) + df_new = df_api + if self.verbose: + print(f"[{self.tag}] entries from DB after upsert: {len(df_new)}") + + # 7) Pull any DB-only records we didn’t fetch locally + df_db = pd.DataFrame(columns=df_csv.columns) + if self.use_penguin: + missing_db = seen_db - seen_local + if missing_db: + # search by DOI prefix + df_id = penguin.id_search( + ids=[f"doi:{doi}" for doi in missing_db], + as_pandas=True + ) + df_db = df_id[df_id.s2id.notnull()].reset_index(drop=True) + if self.verbose: + print(f"[{self.tag}] added DB-only records: {len(df_db)}") + + # 8) Merge everything, dedupe on the S2 UID, save + df_all = pd.concat([df_csv, df_new, df_db], ignore_index=True) + df_all = df_all.drop_duplicates(subset=["s2id"]).reset_index(drop=True) + df_all.to_csv(csv_path, index=False) + if self.verbose: + print(f"[{self.tag}] total after merge: {len(df_all)}") + + # 9) Move cache to real cache dir if requested + real_cache = self.init_settings.get("name") + if real_cache: + copied = copy_all_files(cache_dir, real_cache, preserve_metadata=True) + if self.verbose: + print(f"[{self.tag}] copied {copied} files to real cache at {real_cache}") + + + # 10) Checkpoint & attach + self.register_checkpoint(self.provides[0], str(csv_path)) + bundle[f"{self.tag}.{self.provides[0]}"] = df_all diff --git a/TELF/pipeline/blocks/sample_matrix_mask_block.py b/TELF/pipeline/blocks/sample_matrix_mask_block.py new file mode 100644 index 00000000..1a0f45ac --- /dev/null +++ b/TELF/pipeline/blocks/sample_matrix_mask_block.py @@ -0,0 +1,130 @@ +# blocks/sample_matrix_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import numpy as np +import pandas as pd +from sklearn.model_selection import train_test_split + +from .base_block import AnimalBlock +from .data_bundle import DataBundle + + +# ------------------------------------------------------------------ # +# helper # +# ------------------------------------------------------------------ # +def _sample_matrix( + X: np.ndarray, + *, + sample_ratio: float = 0.05, + random_state: int = 42, + stratify: bool = True, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """ + Re-implements the original `sample_matrix` helper. + + Returns + ------- + X_orig : original matrix (float32) + X_train : matrix with sampled entries set to 0 (float32) + mask : binary mask (1 = kept, 0 = removed) (int8) + removed_coords : coordinates of removed elements, shape (N, 2) (int32) + """ + flat_matrix = X.flatten() + indices = np.arange(flat_matrix.size) + + # choose indices to remove + if stratify: + _, sampled_idx = train_test_split( + indices, + test_size=sample_ratio, + stratify=flat_matrix, + random_state=random_state, + ) + else: + _, sampled_idx = train_test_split( + indices, + test_size=sample_ratio, + random_state=random_state, + ) + + # create masked copy + X_train = X.astype(float).copy() + np.put(X_train, sampled_idx, np.nan) + + removed_coords = np.argwhere(np.isnan(X_train)) + mask = np.ones_like(X_train, dtype=int) + mask[tuple(removed_coords.T)] = 0 + X_train[np.isnan(X_train)] = 0 # replace NaN so downstream code is safe + + return ( + X.astype("float32"), + X_train.astype("float32"), + mask.astype("int8"), + removed_coords.astype("int32"), + ) + + +# ------------------------------------------------------------------ # +# block # +# ------------------------------------------------------------------ # +class SampleMatrixMaskBlock(AnimalBlock): + """ + Randomly masks a proportion of entries in a binary matrix. + + needs : ('X',) + provides : ('X_train', 'MASK', 'removed_coords') + tag : 'SampleMatrix' + """ + + CANONICAL_NEEDS = ("X",) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("X_train", "MASK", "removed_coords",), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "SampleMatrix", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + default_init = {"verbose": True} + default_call = { + "sample_ratio": 0.05, + "random_state": 42, + "stratify": True, + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # Load matrix (path → DataFrame/ndarray or in-memory) + X = self.load_path(bundle[self.needs[0]]) + + # Sample / mask + X_orig, X_train, mask, removed_coords = _sample_matrix(X, **self.call_settings) + + # Store outputs under this block’s namespace + ns = self.tag + bundle[f"{ns}.X"] = X_orig + bundle[f"{ns}.X_train"] = X_train + bundle[f"{ns}.MASK"] = mask + bundle[f"{ns}.removed_coords"] = removed_coords diff --git a/TELF/pipeline/blocks/sbatch_block.py b/TELF/pipeline/blocks/sbatch_block.py new file mode 100644 index 00000000..a349dcde --- /dev/null +++ b/TELF/pipeline/blocks/sbatch_block.py @@ -0,0 +1,182 @@ +# blocks/sbatch_block.py +from __future__ import annotations +import subprocess, textwrap +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import jsonpickle +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +# ───────────────────────── helpers ────────────────────────── + +def _nospace(s: str) -> str: + """Make a safe SLURM job-name / path fragment by replacing spaces.""" + return s.replace(" ", "_") + +def clean_path(p: Path | str) -> Path: + """Recursively apply _nospace to every component of the path.""" + p = Path(p) + return Path(*(_nospace(part) for part in p.parts)) + +def shquote(x: str | Path) -> str: # cheap, POSIX-only quoting + return f'"{x}"' + +# Default SLURM flags; the caller may override any of them via slurm_config +DEFAULT_SLURM: Dict[str, str] = { + "partition": "production", + "output": "slurm-%j.out", +} + + +class SBatchBlock(AnimalBlock): + """ + Wrap *any* AnimalBlock and run it via `sbatch`. + + Behaviour + --------- + • **Skip submission** if every declared checkpoint already exists. + • Otherwise: + 1. create a staging dir `/` (spaces → `_`); + 2. serialize the current bundle + wrapped block there; + 3. write a `run_block.py` runner (faulthandler enabled); + 4. write a hardened `run.sbatch` (threads pinned, Arrow off); + 5. `sbatch run.sbatch` and `sys.exit` so the pipeline resumes later. + """ + def __init__( + self, + wrapped_block: AnimalBlock, + *, + venv_type: str = "conda", # "conda" | "venv" | "poetry" + venv_path: str = "TELF", # env name or path + slurm_config: Dict[str, str] | None = None, + needs: Sequence[str] = (), + provides: Sequence[str] = (), + # tag: str = "SBatchWrapper", + conditional_needs: Sequence[Tuple[str, Any]] = (), + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + **kw, + ) -> None: + tag=wrapped_block.tag + super().__init__( + needs=needs or wrapped_block.needs, + provides=provides or wrapped_block.provides, + tag=tag, + conditional_needs=conditional_needs, + init_settings=init_settings or {}, + call_settings=call_settings or {}, + **kw, + ) + self.wrapped_block = wrapped_block + self.venv_type = venv_type.lower() + self.venv_path = venv_path + self.slurm = {**DEFAULT_SLURM, **(slurm_config or {})} + + def run(self, bundle: DataBundle) -> None: + # 1) fast-skip on existing checkpoints + ck_keys = getattr(self.wrapped_block, "checkpoint_keys", ()) + if ck_keys and all( + (f"{self.wrapped_block.tag}.{ck}") in bundle and + Path(bundle[f"{self.wrapped_block.tag}.{ck}"]).exists() + for ck in ck_keys + ): + print(f"⏭ {self.tag}: existing checkpoints – skipping submission.") + try: + self.wrapped_block._after_checkpoint_skip(bundle) + except AttributeError: + pass + return + + # 2) staging directory + base_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]).resolve() + workdir = clean_path(base_dir / self.tag) + workdir.mkdir(parents=True, exist_ok=True) + + # adjust SLURM output path + orig_out = self.slurm.get("output", DEFAULT_SLURM["output"]) + self.slurm["output"] = str(workdir / orig_out) + + # 2b) resume if runner already completed + done = workdir / "_complete.json" + if done.exists(): + # runner wrote out a small dict: { provide_name: value, ... } + resumed: dict = jsonpickle.decode(done.read_text()) + wrapped_tag = self.wrapped_block.tag + for p in self.provides: + ns = f"{wrapped_tag}.{p}" + if ns not in bundle: + bundle[ns] = resumed[p] + return + + # ensure SAVE_DIR is absolute + bundle[SAVE_DIR_BUNDLE_KEY] = str(base_dir) + + # 3) serialize bundle + block + (workdir / "input_bundle.json").write_text( + jsonpickle.encode(bundle), "utf-8" + ) + (workdir / "block.json").write_text( + jsonpickle.encode(self.wrapped_block), "utf-8" + ) + + # 4) runner script + runner = workdir / "run_block.py" + runner.write_text(textwrap.dedent(f""" + import faulthandler, jsonpickle + faulthandler.enable() + + bundle = jsonpickle.decode(open(r"{workdir/'input_bundle.json'}").read()) + block = jsonpickle.decode(open(r"{workdir/'block.json'}").read()) + out_bundle = block(bundle) + + # collect just the wrapped block's provides + result = {{}} + for p in {list(self.provides)!r}: + result[p] = out_bundle[f"{{block.tag}}.{{p}}"] + + open(r"{done}", "w").write(jsonpickle.encode(result)) + """).strip(), "utf-8") + + # 5) sbatch script + sbatch = workdir / "run.sbatch" + lines = [ + "#!/bin/bash", + f"#SBATCH --job-name={_nospace(self.tag)}", + *(f"#SBATCH --{k}={v}" for k, v in self.slurm.items()), + "", + "ulimit -c unlimited", + "export OMP_NUM_THREADS=1", + "export OPENBLAS_NUM_THREADS=1", + "export MKL_NUM_THREADS=1", + "export NUMEXPR_NUM_THREADS=1", + "export MKL_THREADING_LAYER=GNU", + "export OPENBLAS_DISABLE_THREADS=1", + "export PANDAS_ARROW_DISABLED=1", + "", + f"cd {shquote(workdir)}", + ] + + match self.venv_type: + case "venv": + lines.append(f"source {shquote(Path(self.venv_path).resolve()/ 'bin'/'activate')}") + case "conda": + lines += [ + "source $(conda info --base)/etc/profile.d/conda.sh", + f"conda activate {self.venv_path}", + ] + case "poetry": + lines += [ + f"cd {shquote(Path.cwd())}", + "poetry install --no-root --quiet", + ] + case _: + raise ValueError(f"Unknown venv_type {self.venv_type!r}") + + lines.append(f"python -Xfaulthandler {shquote(runner)}") + sbatch.write_text("\n".join(lines), "utf-8") + + # 6) submit and exit + print(f"🚀 {self.tag}: submitting via sbatch …") + subprocess.run(["sbatch", str(sbatch)], check=True) + raise SystemExit(f"{self.tag} submitted – pipeline will resume after job finishes.") diff --git a/TELF/pipeline/blocks/scopus_block.py b/TELF/pipeline/blocks/scopus_block.py new file mode 100644 index 00000000..306fe6a0 --- /dev/null +++ b/TELF/pipeline/blocks/scopus_block.py @@ -0,0 +1,157 @@ +from pathlib import Path +from typing import Dict, Sequence, Any +import pandas as pd +import nest_asyncio +nest_asyncio.apply() +from pymongo import ASCENDING + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...pre_processing import Scopus +from ...applications.Penguin import Penguin + +from ...helpers.file_system import copy_all_files + +class ScopusBlock(AnimalBlock): + CANONICAL_NEEDS = ('df', ) + + def __init__( + self, + use_penguin: bool = False, + penguin_settings: Dict[str, Any] = { + "uri": "localhost:27017", + "db_name": "Penguin", + "username": None, + "password": None, + }, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("df",), + tag: str = "Scopus", + conditional_needs: Sequence[tuple[str, Any]] = (), + *, + init_settings: Dict[str, Any] = None, + call_settings: Dict[str, Any] = None, + **kw, + ) -> None: + self.use_penguin = use_penguin + self.penguin_settings = penguin_SETTINGS = penguin_settings + + default_init = { + 'keys': None, + 'mode': 'fs', + 'name': None, # real cache dir if provided + 'ignore': None, + 'verbose': True, + } + super().__init__( + needs=needs, + provides=provides, + init_settings=self._merge(default_init, init_settings), + call_settings={}, + conditional_needs=conditional_needs, + tag=tag, + **kw, + ) + + def run(self, bundle: DataBundle) -> None: + # 1) Load input & extract DOIs + raw = bundle[self.needs[0]] + df_in = self.load_path(raw) if isinstance(raw, (str, Path)) else raw.copy() + dois = df_in.doi.dropna().astype(str).tolist() + if self.verbose: + print(f"[{self.tag}] candidate DOIs count: {len(dois)}") + + # 2) Prepare output dirs & CSV + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(parents=True, exist_ok=True) + csv_path = out_dir / 'scopus.csv' + cache_dir = out_dir / 'SCOPUS_CACHE' + cache_dir.mkdir(exist_ok=True, parents=True) + if self.verbose: + print(f"[{self.tag}] cache dir: {cache_dir}") + + # 3) Connect to Penguin if requested + if self.use_penguin: + penguin = Penguin(**self.penguin_settings) + doi_field = penguin.scopus_attributes['doi'] + col = penguin.db[Penguin.SCOPUS_COL] + col.create_index([(doi_field, ASCENDING)]) + seen_db = set(col.distinct(doi_field, {doi_field: {'$in': dois}})) + else: + seen_db = set() + if self.verbose: + print(f"[{self.tag}] seen in Mongo: {len(seen_db)}") + + # 4) Load or initialize local CSV + if csv_path.exists(): + df_csv = pd.read_csv(csv_path) + else: + df_csv = pd.DataFrame(columns=df_in.columns) + seen_local = set(df_csv.doi.dropna().astype(str).tolist()) + if self.verbose: + print(f"[{self.tag}] seen locally: {len(seen_local)}") + + # 5) Figure out what to fetch from Scopus API + to_fetch = [d for d in dois if d not in seen_local and d not in seen_db] + if self.verbose: + print(f"[{self.tag}] to fetch count: {len(to_fetch)}") + + # 6) Fetch any new records + df_new = pd.DataFrame(columns=df_csv.columns) + if to_fetch: + settings = dict(self.init_settings, name=str(cache_dir)) + sc = Scopus(**settings) + query = ' OR '.join(f"DOI({doi})" for doi in to_fetch) + try: + df_new, _ = sc.search(query) + except ValueError as e: + if "No Scopus papers" in str(e): + df_new = pd.DataFrame(columns=df_csv.columns) + else: + raise + if self.verbose: + print(f"[{self.tag}] fetched via API: {len(df_new)}") + + # 6a) upsert those JSONs into Penguin + if self.use_penguin and not df_new.empty: + penguin.add_many_documents(cache_dir, source='Scopus', overwrite=True) + if self.verbose: + print(f"[{self.tag}] bulk-upserted JSON files into Mongo") + + # 6b) re-load only the ones we just added + df_id = penguin.id_search( + ids=[f"doi:{doi}" for doi in to_fetch], + as_pandas=True + ) + id_key = penguin.scopus_attributes['id'] + df_new = df_id[df_id[id_key].notnull()].reset_index(drop=True) + if self.verbose: + print(f"[{self.tag}] entries from DB after upsert: {len(df_new)}") + + # 7) Pull any DB-only records we didn’t fetch locally + df_db = pd.DataFrame(columns=df_csv.columns) + if self.use_penguin: + missing_db = seen_db - seen_local + if missing_db: + docs = list(col.find({doi_field: {'$in': list(missing_db)}})) + df_db = pd.DataFrame(docs).drop(columns=['_id'], errors='ignore') + if self.verbose: + print(f"[{self.tag}] added DB-only records: {len(df_db)}") + + # 8) Merge, dedupe, save, move cache, checkpoint + df_all = pd.concat([df_csv, df_new, df_db], ignore_index=True) + df_all = df_all.drop_duplicates(subset=['doi']).reset_index(drop=True) + df_all.to_csv(csv_path, index=False) + if self.verbose: + print(f"[{self.tag}] total after merge: {len(df_all)}") + + # 9) Optionally relocate the flat‐file cache into a “real” cache dir + real_cache = self.init_settings.get('name') + if real_cache: + copied = copy_all_files(cache_dir, real_cache, preserve_metadata=True) + if self.verbose: + print(f"[{self.tag}] copied {copied} files to real cache at {real_cache}") + + + self.register_checkpoint(self.provides[0], str(csv_path)) + bundle[f"{self.tag}.{self.provides[0]}"] = df_all diff --git a/TELF/pipeline/blocks/scores_block.py b/TELF/pipeline/blocks/scores_block.py new file mode 100644 index 00000000..46e16e07 --- /dev/null +++ b/TELF/pipeline/blocks/scores_block.py @@ -0,0 +1,127 @@ +# blocks/compute_scores_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple, Optional + +import numpy as np +import pandas as pd +from .base_block import AnimalBlock +from .data_bundle import DataBundle + +from sklearn.metrics import ( + root_mean_squared_error, + f1_score, + precision_score, + recall_score, + auc, + accuracy_score, + matthews_corrcoef, + precision_recall_curve, + roc_curve, +) + +# ------------------------------------------------------------------ # +# helper # +# ------------------------------------------------------------------ # +def _compute_scores( + y_true: np.ndarray | pd.Series | list, + y_pred: np.ndarray | pd.Series | list, + *, + binary: bool = True, +) -> Dict[str, float | None]: + """ + Replicates the original `get_scores` function and returns a dict of metrics. + """ + y_true = np.asarray(y_true) + y_pred = np.asarray(y_pred) + + # ROC / PR curves expect probabilities; we fall back to labels gracefully. + fpr, tpr, _ = roc_curve(y_true, y_pred, pos_label=1) + roc_auc = auc(fpr, tpr) + + precision_line, recall_line, _ = precision_recall_curve( + y_true, y_pred, pos_label=1 + ) + pr_auc = auc(recall_line, precision_line) + + rmse = root_mean_squared_error(y_true, y_pred) + + if binary: + acc = accuracy_score(y_true, y_pred) + f1 = f1_score(y_true, y_pred, average="weighted") + precision = precision_score(y_true, y_pred, average="weighted") + recall = recall_score(y_true, y_pred, average="weighted") + mcc = matthews_corrcoef(y_true, y_pred) + else: + acc = f1 = precision = recall = mcc = None + + return { + "roc_auc": roc_auc, + "pr_auc": pr_auc, + "accuracy": acc, + "f1": f1, + "precision": precision, + "recall": recall, + "mcc": mcc, + "rmse": rmse, + } + + +# ------------------------------------------------------------------ # +# block # +# ------------------------------------------------------------------ # +class ComputeScoresBlock(AnimalBlock): + """ + Compute evaluation metrics for model predictions. + + needs : ('y_true', 'y_pred',) + provides : ('scores', 'scores_df',) + tag : 'ComputeScores' + conditional : none + """ + CANONICAL_NEEDS = ("y_true", "y_pred",) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("scores",), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "ComputeScores", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + default_init = {"verbose": True} + default_call = { + "binary": True, + } + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + # Pull y_true and y_pred (either paths or in-memory arrays/Series) + y_true = self.load_path(bundle[self.needs[0]]) + y_pred = self.load_path(bundle[self.needs[1]]) + + # Compute scores + scores_dict = _compute_scores(y_true, y_pred, **self.call_settings) + + # Store: as dict and as single-row DataFrame for convenience + bundle[f"{self.tag}.{self.provides[0]}"] = scores_dict + bundle[f"{self.tag}.{self.provides[0]}_df"] = pd.DataFrame([scores_dict]) diff --git a/TELF/pipeline/blocks/semantic_hnmfk_block.py b/TELF/pipeline/blocks/semantic_hnmfk_block.py new file mode 100644 index 00000000..eea6412f --- /dev/null +++ b/TELF/pipeline/blocks/semantic_hnmfk_block.py @@ -0,0 +1,185 @@ +# blocks/semantic_hnmfk_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple, Optional + +import numpy as np +import pandas as pd +import scipy.sparse as ss + +from ...factorization import HNMFk +from ...pre_processing import Beaver +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +# ------------------------------------------------------------------ # +# Helper – callback to rebuild a doc-word matrix at every node # +# ------------------------------------------------------------------ # +class CustomSemanticCallback: + def __init__( + self, + df: pd.DataFrame, + vocabulary: list, + *, + target_column: str = "clean_title_abstract", + options: Optional[Dict[str, Any]] = None, + matrix_type: str = "tfidf", + ) -> None: + self.df = df + self.vocabulary = vocabulary + self.target_column = target_column + self.options = options or {"min_df": 5, "max_df": 0.5} + self.options["vocabulary"] = self.vocabulary + self.matrix_type = matrix_type + + def __call__(self, original_idx: np.ndarray): + sub_df = ( + self.df.iloc[original_idx] + .copy(deep=True) + .reset_index(drop=True) + ) + + # HARD RESET of Arrow-backed strings + sub_df.columns = pd.Index([str(c) for c in sub_df.columns], dtype="object") + if "string" in str(sub_df[self.target_column].dtype): + sub_df[self.target_column] = sub_df[self.target_column].astype("object") + + beaver = Beaver() + try: + X, vocab = beaver.documents_words( + dataset=sub_df, + target_column=self.target_column, + options=self.options, + matrix_type=self.matrix_type, + save_path=None, + ) + return X.T.tocsr(), {"vocab": vocab} + except Exception: + return ss.csr_matrix([[1]]), { + "stop_reason": "documents_words could not build matrix", + } + + +# ------------------------------------------------------------------ # +# Main block # +# ------------------------------------------------------------------ # +class SemanticHNMFkBlock(AnimalBlock): + CANONICAL_NEEDS = ("X", "df", "vocabulary") + + def __init__( + self, + *, + col: str = "clean_title_abstract", + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("model", "model_path"), + checkpoint_keys=("model_path",), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "SemanticHNMFk", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kw: Any, + ) -> None: + self.col = col + + # ---- defaults -------------------------------------------------- + nmfk_params = { + "n_perturbs": 2, + "n_iters": 2, + "epsilon": 0.015, + "n_jobs": -1, + "init": "nnsvd", + "use_gpu": False, + "save_output": True, + "collect_output": True, + "predict_k_method": "sill", + "verbose": True, + "nmf_verbose": False, + "transpose": False, + "sill_thresh": 0.8, + "pruned": True, + "nmf_method": "nmf_fro_mu", + "calculate_error": True, + "predict_k": True, + "use_consensus_stopping": 0, + "calculate_pac": True, + "consensus_mat": True, + "perturb_type": "uniform", + "perturb_multiprocessing": False, + "perturb_verbose": False, + "simple_plot": True, + "k_search_method": "bst_pre", + "H_sill_thresh": 0.1, + "clustering_method": "kmeans", + "device": -1, + } + + default_init = { + "nmfk_params": [nmfk_params], + "cluster_on": "H", + "depth": 1, + "sample_thresh": 5, + "K2": False, + "Ks_deep_min": 1, + "Ks_deep_max": 20, + "Ks_deep_step": 1, + "random_identifiers": False, + "root_node_name": "Root", + } + + default_call = { + "Ks": range(1, 21), + "from_checkpoint": True, + "save_checkpoint": True, + } + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + checkpoint_keys=checkpoint_keys, + **kw, + ) + + def run(self, bundle: DataBundle) -> None: + # 1 ▸ load input matrix + X_val = bundle[self.needs[0]] + X = self.load_path(X_val) if isinstance(X_val, (str, Path)) else X_val + + # 2 ▸ assemble HNMFk kwargs + init_cfg = dict(self.init_settings) + if "experiment_name" not in init_cfg and SAVE_DIR_BUNDLE_KEY in bundle: + init_cfg["experiment_name"] = ( + Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + ) + + init_cfg["generate_X_callback"] = CustomSemanticCallback( + df=bundle[self.needs[1]], + vocabulary=bundle[self.needs[2]], + target_column=self.col, + ) + + # 3 ▸ fit hierarchy & checkpoint + model = HNMFk(**init_cfg) + model.fit(X, **self.call_settings) + + # 4 ▸ persist only the path in the checkpoint and in the bundle + model_dir = model.experiment_save_path + self.register_checkpoint(self.provides[1], model_dir) + bundle[f"{self.tag}.{self.provides[1]}"] = model_dir + + # 5 ▸ immediately re-load the model so 'model' is always provided + reloaded = HNMFk(experiment_name=model_dir) + reloaded.load_model() + bundle[f"{self.tag}.{self.provides[0]}"] = reloaded + + def _after_checkpoint_skip(self, bundle: DataBundle) -> None: + # when re‐running from an existing checkpoint, load into memory + model_dir = bundle[f"{self.tag}.{self.provides[1]}"] + model = HNMFk(experiment_name=model_dir) + model.load_model() + bundle[f"{self.tag}.{self.provides[0]}"] = model diff --git a/TELF/pipeline/blocks/split_block.py b/TELF/pipeline/blocks/split_block.py new file mode 100644 index 00000000..1b670d46 --- /dev/null +++ b/TELF/pipeline/blocks/split_block.py @@ -0,0 +1,118 @@ +# blocks/split_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +from .base_block import AnimalBlock +from .data_bundle import DataBundle +from ...factorization import SPLIT + + +class SPLITBlock(AnimalBlock): + """ + Wrapper around TELF.factorization.SPLIT. + + needs + ----- + ('Xs',) – sparse or dense *collection* of matrices (see SPLIT docs) + + provides + -------- + ('results', 'model') + + * **results** – output of `model.transform()` + * **model** – the fitted SPLIT instance itself + + tag + --- + 'SPLIT' + """ + + CANONICAL_NEEDS = ("Xs",) + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("results", "model"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "SPLIT", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + # ---------- defaults that feed straight into SPLIT(...) -------- # + params = { + "n_perturbs": 10, + "n_iters": 100, + "epsilon": 0.015, + "n_jobs": 1, + "init": "nnsvd", + "use_gpu": False, + "save_output": True, + "verbose": False, + "transpose": False, + "sill_thresh": 0.9, + "nmf_method": "nmf_fro_mu", + } + + Ks = { + "X1": range(1, 9), + "X2": range(1, 10), + "X3": range(1, 11), + } + nmfk_params = {name: params for name in Ks} + + default_init = { + "Ks": Ks, + "nmfk_params": nmfk_params, + "split_nmfk_params": params, + "Ks_split_step": 1, + "Ks_split_min": 1, + "H_regress_gpu": False, + "H_learn_method": "regress", + "H_regress_iters": 1000, + "H_regress_method": "fro", + "H_regress_init": "MitH", + "verbose": True, + "random_state": 42, + } + + # ---------- args to `model.fit(**call_settings)` ---------------- # + default_call: Dict[str, Any] = {} + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + src = bundle[self.needs[0]] + if isinstance(src, (str, Path)): + Xs = self.load_path(src) + else: + Xs = src + + model = SPLIT(Xs=Xs, **self.init_settings) + model.fit(**self.call_settings) + + results = model.transform() + + ns = self.tag + bundle[f"{ns}.{self.provides[0]}"] = results + bundle[f"{ns}.{self.provides[1]}"] = model diff --git a/TELF/pipeline/blocks/split_transfer_block.py b/TELF/pipeline/blocks/split_transfer_block.py new file mode 100644 index 00000000..56bf4a21 --- /dev/null +++ b/TELF/pipeline/blocks/split_transfer_block.py @@ -0,0 +1,149 @@ +# blocks/split_transfer_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Sequence, Tuple + +import numpy as np +import pandas as pd + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...factorization import SPLITTransfer + +class SPLITTransferBlock(AnimalBlock): + """ + Pipeline block that wraps :class:`TELF.factorization.SPLITTransfer`. + + needs + ----- + ('X_known', 'X_target', 'indicator') + + * **X_known** – source-domain matrix (sparse or dense, or a path) + * **X_target** – target-domain matrix (same format as above) + * **indicator** – boolean / 0-1 array indicating the held-out + positions to predict in *X_target* + + provides + -------- + ('model', 'y_pred_train', 'y_pred_test') + + * **model** – fitted :class:`SPLITTransfer` + * **y_pred_train** – predictions (source/target) for the *train* mask + * **y_pred_test** – predictions for the *test* mask + + tag + --- + 'SPLITTransfer' + """ + + CANONICAL_NEEDS = ("X_known", "X_target", "indicator") + + # ------------------------------------------------------------------ # + # constructor # + # ------------------------------------------------------------------ # + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("model", "y_pred_train", "y_pred_test"), + conditional_needs: Sequence[Tuple[str, Any]] = (), + tag: str = "SPLITTransfer", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + verbose: bool = True, + **kwargs: Any, + ) -> None: + + # ------------------- base hyper-parameters -------------------- # + _base_known = { + "n_perturbs": 30, + "n_iters": 1000, + "epsilon": 0.015, + "n_jobs": -1, + "init": "nnsvd", + "use_gpu": False, + "save_path": None, + "save_output": True, + "sill_thresh": 0.5, + "nmf_method": "nmf_fro_mu", + } + _base_target = {**_base_known, "epsilon": 0.01, "sill_thresh": 0.8} + _base_split = {**_base_known, "sill_thresh": 0.6} + + default_init = { + # search ranges + "Ks_known": range(1, 10), # 1 … 9 + "Ks_target": range(1, 10), + "Ks_split_step": 1, + "Ks_split_min": 1, + # regression hyper-params + "H_regress_gpu": False, + "H_learn_method": "regress", + "H_regress_init": "MitH", + "H_regress_iters": 1000, + "H_regress_method": "fro", + # nested NMFk parameters + "nmfk_params_known": _base_known, + "nmfk_params_target": _base_target, + "nmfk_params_split": _base_split, + # transfer-learning settings + "transfer_method": "SVR", # or "model" + "transfer_regress_params": {}, + "transfer_model": None, # supply if transfer_method == "model" + # misc + "verbose": True, + "random_state": 42, + } + + # forwarded to model.fit / transform / predict if ever needed + default_call: Dict[str, Any] = {} + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + verbose=verbose, + **kwargs, + ) + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def _to_numpy(self, obj: Any) -> np.ndarray: + """Utility: convert DataFrame / Series to ndarray, keep ndarray else.""" + if isinstance(obj, (str, Path)): + obj = self.load_path(obj) + if isinstance(obj, pd.DataFrame): + return obj.values + if isinstance(obj, pd.Series): + return obj.to_numpy() + return np.asarray(obj) + + def run(self, bundle: DataBundle) -> None: + # 1️⃣ Gather inputs ------------------------------------------------- # + X_known_src, X_target_src, indicator_src = ( + bundle[n] for n in self.needs + ) + + X_known = self._to_numpy(X_known_src) + X_target = self._to_numpy(X_target_src) + indicator = self._to_numpy(indicator_src) + + # 2️⃣ Fit SPLITTransfer ------------------------------------------- # + model = SPLITTransfer(**self.init_settings) + model.fit(X_known, X_target) + model.transform(indicator) + + # 3️⃣ Predictions -------------------------------------------------- # + y_pred_train = model.predict(test=False) + y_pred_test = model.predict(test=True) + + # 4️⃣ Store outputs ------------------------------------------------ # + ns = self.tag + bundle[f"{ns}.{self.provides[0]}"] = model + bundle[f"{ns}.{self.provides[1]}"] = y_pred_train + bundle[f"{ns}.{self.provides[2]}"] = y_pred_test diff --git a/TELF/pipeline/blocks/squirrel_block.py b/TELF/pipeline/blocks/squirrel_block.py new file mode 100644 index 00000000..b954a507 --- /dev/null +++ b/TELF/pipeline/blocks/squirrel_block.py @@ -0,0 +1,97 @@ +from pathlib import Path +from typing import Dict, Sequence, Any, Tuple +import pandas as pd + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +from ...pre_processing import Squirrel +from ...pre_processing.Squirrel.pruners import EmbeddingPruner +from ...pre_processing.Squirrel.pruners import LLMPruner + + +class SquirrelBlock(AnimalBlock): + CANONICAL_NEEDS = ('df', ) + + def __init__( + self, + needs = CANONICAL_NEEDS, + provides = ("df",), + tag: str = "Squirrel", + conditional_needs: Sequence[Tuple[str, Any]] = (), + *, + init_settings: Dict[str, Any] = None, + call_settings: Dict[str, Any] = None, + **kw, + ) -> None: + + emb_pruner = EmbeddingPruner( + embedding_model="SPECTER", + distance_std_factor=5.0, + overwrite_embeddings=False, + use_gpu=True, + verbose=True, + ) + llm_pruner = LLMPruner( + llm_model_name="llama3.2:latest", + llm_api_url="http://localhost:11434", + llm_vote_trials=4, + llm_promote_threshold=0.75, + llm_temperature=0.7, + verbose=True, + ) + self.pipeline = [emb_pruner, llm_pruner] + self.backup_pipeline = [llm_pruner] + default_init = { + 'data_column': 'title_abstract', + 'label_column': 'type', + 'reference_label': 0, + 'aggregrate_prune': True, + 'pipeline':self.pipeline + } + default_call = {} + + super().__init__( + needs=needs, + provides=provides, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + conditional_needs=conditional_needs, + **kw, + ) + + def run(self, bundle: DataBundle) -> None: + raw = bundle[self.needs[0]] + if isinstance(raw, (str, Path)): + df = self.load_path(raw) + else: + df = raw.copy() + + # If there are less than 25 documents, use only llm pipeline + label_col = self.init_settings['label_column'] + ref_label = self.init_settings['reference_label'] + count_ref = int((df[label_col] == ref_label).sum()) + if count_ref < 25: + self.init_settings['pipeline'] = self.backup_pipeline + + if self.init_settings['data_column'] not in df.columns: + try: + # df[self.init_settings['data_column']] = df['title'] + ' ' + df['abstract'] + df[self.init_settings['data_column']] = df['title'].astype(str) + ' ' + df['abstract'].astype(str) + except KeyError: + raise ValueError(f"Data column {self.init_settings['data_column']} not found in DataFrame.") + except Exception as e: + raise RuntimeError(f"An error occurred while creating the data column: {e}") + + squirrel_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + squirrel = Squirrel(data_source=df, + output_dir=squirrel_dir, + **self.init_settings) + squirrel() + expected_csv = squirrel_dir / 'squirrel_pruned.csv' + + result_df = pd.read_csv(expected_csv) + self.register_checkpoint(self.provides[0], expected_csv) + + bundle[f"{self.tag}.{self.provides[0]}"] = result_df diff --git a/TELF/pipeline/blocks/term_attribution_block.py b/TELF/pipeline/blocks/term_attribution_block.py new file mode 100644 index 00000000..472dcd66 --- /dev/null +++ b/TELF/pipeline/blocks/term_attribution_block.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import ast +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union +import warnings + +import pandas as pd +import numpy as np + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...applications.Cheetah import Cheetah +from ...pre_processing.Vulture.tokens_analysis.top_words import get_top_words + + +class TermAttributionBlock(AnimalBlock): + """ + Runs Cheetah-based term representation + attribution on a DataFrame, + including positive and negative dependents as additional terms. + Processes both clean_title_abstract and raw title+abstract, and + generates n-gram files for each, with counts in representation CSV. + """ + CANONICAL_NEEDS = ("df", "terms") + + def __init__( + self, + *, + needs: Tuple[str, ...] = CANONICAL_NEEDS, + provides: Tuple[str, ...] = ("df", "term_representation_df"), + tag: str = "Attribution", + init_settings: Optional[Dict[str, Any]] = None, + call_settings: Optional[Dict[str, Any]] = None, + ): + default_call = { + "text_col": "clean_title_abstract", + "cheetah_verbose": False, + "ngram_top_n": 100 + } + super().__init__( + needs=needs, + provides=provides, + tag=tag, + init_settings=init_settings or {}, + call_settings={**default_call, **(call_settings or {})}, + ) + + def run(self, bundle: DataBundle) -> None: + # 1) Load inputs + df: pd.DataFrame = self.load_path(bundle[self.needs[0]]) + df = df.reset_index(drop=True) + raw_terms = self.load_path(bundle[self.needs[1]]) + + # 2) Parse main_terms and dependents + main_terms: List[str] = [] + pos_terms: set[str] = set() + neg_terms: set[str] = set() + + def extract_entry(val: Any) -> None: + if isinstance(val, str): + try: + parsed = ast.literal_eval(val) + except Exception: + main_terms.append(val) + return + if isinstance(parsed, dict) and len(parsed) == 1: + term, spec = next(iter(parsed.items())) + else: + main_terms.append(val) + return + elif isinstance(val, dict) and len(val) == 1: + term, spec = next(iter(val.items())) + else: + raise TypeError(f"Unsupported term entry: {val!r}") + + main_terms.append(term) + if isinstance(spec, dict): + for p in spec.get('positives', []): pos_terms.add(p) + for n in spec.get('negatives', []): neg_terms.add(n) + elif isinstance(spec, list): + for p in spec: pos_terms.add(p) + + if isinstance(raw_terms, pd.DataFrame): + col0 = raw_terms.columns[0] + for v in raw_terms[col0]: extract_entry(v) + else: + for entry in raw_terms: extract_entry(entry) + + # 3) Dedupe & order terms + union_main_pos = set(main_terms) | pos_terms + all_terms = ( + main_terms + + [t for t in pos_terms if t not in main_terms] + + [t for t in neg_terms if t not in union_main_pos] + ) + seen = set() + deduped_terms = [t for t in all_terms if not (t in seen or seen.add(t))] + + # 4) Validate clean text column + text_col = self.call_settings['text_col'] + if text_col not in df.columns: + raise KeyError(f"Missing text column {text_col!r}") + + # 5) Prepare Document Sets + docs_clean = df[text_col].fillna("").astype(str).tolist() + docs_raw = ( + df.get('title', pd.Series(dtype=str)).fillna("") + + " " + df.get('abstract', pd.Series(dtype=str)).fillna("") + ).astype(str).tolist() + N_clean = len(docs_clean) + N_raw = len(docs_raw) + + # 6) If nothing to process, build empty DataFrames + if N_clean == 0 or not deduped_terms: + term_representation_df = pd.DataFrame(columns=[ + 'Term', 'TF_clean', 'DF_clean', 'TF_raw', 'DF_raw' + ]) + attribution_df = pd.DataFrame(columns=list(df.columns) + ['term']) + else: + # 7) Compute TF counts via regex + term_tfs_clean: Dict[str,int] = {} + term_tfs_raw: Dict[str,int] = {} + total_tf_clean = 0 + total_tf_raw = 0 + + for term in deduped_terms: + pat = rf"\b{re.escape(term)}\b" + tf_c = sum(len(re.findall(pat, d, flags=re.IGNORECASE)) for d in docs_clean) + tf_r = sum(len(re.findall(pat, d, flags=re.IGNORECASE)) for d in docs_raw) + term_tfs_clean[term] = tf_c + term_tfs_raw[term] = tf_r + total_tf_clean += tf_c + total_tf_raw += tf_r + + # 8) Index only clean for DF via Cheetah + cheetah_clean = Cheetah(verbose=self.call_settings['cheetah_verbose']) + cheetah_clean.index( + data=df.reset_index(drop=True), + columns={'abstract': text_col}, + verbose=self.call_settings['cheetah_verbose'] + ) + + repr_rows: List[Dict[str,Any]] = [] + attr_rows: List[Dict[str,Any]] = [] + + for term in deduped_terms: + tf_c = term_tfs_clean[term] + tf_r = term_tfs_raw[term] + pat = rf"\b{re.escape(term)}\b" + + # DF_clean via Cheetah.search + df_clean = cheetah_clean.search( + query=[term], and_search=True, + in_title=False, in_abstract=True, + do_results_table=False + )[0] + df_c = len(df_clean) + + # DF_raw via regex count of documents + df_r = sum(1 for d in docs_raw if re.search(pat, d, flags=re.IGNORECASE)) + + repr_rows.append({ + 'Term': term, + 'TF_clean': tf_c, + 'DF_clean': df_c, + 'TF_raw': tf_r, + 'DF_raw': df_r, + }) + + # Attribution: one row per clean match + for idx in df_clean.index: + base = df.iloc[idx].to_dict() + base['term'] = term + attr_rows.append(base) + + term_representation_df = pd.DataFrame.from_records(repr_rows) + attribution_df = pd.DataFrame.from_records(attr_rows) + + # 9) Save CSV outputs + base_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) + out_dir = base_dir / self.tag + out_dir.mkdir(parents=True, exist_ok=True) + path_attr = out_dir / 'df_attribution.csv' + path_repr = out_dir / 'attribution_representation.csv' + attribution_df.to_csv(path_attr, index=False) + term_representation_df.to_csv(path_repr, index=False) + + # 10) Generate n-gram files for both clean & raw + top_n = self.call_settings['ngram_top_n'] + for mode, docs in [('clean', docs_clean), ('raw', docs_raw)]: + for n in (1, 2, 3): + ng = get_top_words(docs, top_n=top_n, n_gram=n, verbose=False) + (out_dir / f'top_{n}-grams_{mode}.csv').write_text( + ng.iloc[:, :3].to_csv(index=False) + ) + + # 11) Register & bundle + self.register_checkpoint(self.provides[0], path_attr) + self.register_checkpoint(self.provides[1], path_repr) + bundle[f"{self.tag}.{self.provides[0]}"] = attribution_df + bundle[f"{self.tag}.{self.provides[1]}"] = term_representation_df diff --git a/TELF/pipeline/blocks/term_search_block.py b/TELF/pipeline/blocks/term_search_block.py new file mode 100644 index 00000000..5330c5f4 --- /dev/null +++ b/TELF/pipeline/blocks/term_search_block.py @@ -0,0 +1,103 @@ +from pathlib import Path +from typing import Dict, Sequence, Any, Tuple +import pandas as pd +import os +import nest_asyncio +nest_asyncio.apply() + +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +from ...pipeline.blocks.base_block import AnimalBlock +from ...helpers.frames import merge_scopus_s2, clean_duplicates +from ...helpers.terms import form_scopus_search, form_s2_search # adjust import to your project layout + +from ...pre_processing.iPenguin.Scopus import Scopus +from ...pre_processing.iPenguin.SemanticScholar import SemanticScholar + +class TermSearchBlock(AnimalBlock): + CANONICAL_NEEDS = ("terms",) + + def __init__( + self, + needs=CANONICAL_NEEDS, + provides=("df",), + tag: str = "TermSearch", + conditional_needs: Sequence[Tuple[str, Any]] = (), + *, + init_settings: Dict[str, Any] = None, + call_settings: Dict[str, Any] = None, + **kw, + ): + default_init = { + 'scopus':{ + 'keys': None, #SCOPUS_KEYS, + 'mode': 'fs', # file system caching mode (default) + 'name': None, #SCOPUS_CACHE, # where to cache the files + 'ignore': None, #scopus_ignore, + 'verbose': True # run high level handler in debug (verbose > 10) + }, + 's2':{ + 'key': None, #S2_KEY, + 'mode': 'fs', # file system caching mode (default) + 'name': None, #S2_CACHE, # where to cache the files + 'ignore': None, #scopus_ignore, + 'verbose': True # run high level handler in debug (verbose > 10) + } + } + default_call = {} + + super().__init__( + needs = needs, + provides = provides, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + conditional_needs=conditional_needs, + tag=tag, + **kw, + ) + + def run(self, bundle:DataBundle) -> None: + result_path = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + result_path.mkdir(parents=True, exist_ok=True) + + terms = self.load_path(bundle[self.needs[0]]) + + # === Step 1: Scopus search via TITLE-ABS + scopus_outfile = result_path / "scopus.csv" + if not scopus_outfile.exists(): + scopus = Scopus(**self.init_settings['scopus']) + queries = [f"({form_scopus_search(term, code='TITLE-ABS')})" for term in terms] + for q in queries: + print(q) + scopus_query = " OR ".join(queries) + scopus_df, _ = scopus.search(scopus_query, n=0) + scopus_df.to_csv(scopus_outfile, index=False) + else: + scopus_df = pd.read_csv(scopus_outfile) + + scopus_df = clean_duplicates(scopus_df) + + # === Step 2: S2 search + s2_outfile = result_path / "s2.csv" + if not s2_outfile.exists(): + s2 = SemanticScholar(**self.init_settings['s2']) + s2_dfs = [] + for term in terms: + q = form_s2_search(term) + tmp_df, _ = s2.search(q, mode="bulk", n=0) + s2_dfs.append(tmp_df) + + s2_df = pd.concat(s2_dfs, ignore_index=True) + s2_df = s2_df.drop_duplicates(subset=["s2id"]).reset_index(drop=True) + s2_df.to_csv(s2_outfile, index=False) + else: + s2_df = pd.read_csv(s2_outfile, lineterminator="\n") + + # === Step 3: Merge + merged_df = merge_scopus_s2(scopus_df, s2_df) + + # === Save final + final_outfile = result_path / "final_papers.csv" + merged_df.to_csv(final_outfile, index=False) + + bundle[f"{self.tag}.{self.provides[0]}"] = merged_df diff --git a/TELF/pipeline/blocks/uniqueness_block.py b/TELF/pipeline/blocks/uniqueness_block.py new file mode 100644 index 00000000..36521b1f --- /dev/null +++ b/TELF/pipeline/blocks/uniqueness_block.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable, Dict, Sequence, List, Optional, Union +import pandas as pd +import ast + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + + +def _nested_dict_splitter(s: Any) -> Dict[str, List[Any]]: + """ + Generic splitter for JSON-like strings representing nested dictionaries. + + Returns a dict mapping: + - '' (empty string) -> list of top-level keys + - each nested key -> flattened list of all values under that key + """ + try: + d = ast.literal_eval(s) if isinstance(s, str) else {} + except Exception: + return {} + + out: Dict[str, List[Any]] = {'': []} + for top_key, val in d.items(): + # record the top-level key + out[''].append(top_key) + + # if the value is a dict, record its nested items + if isinstance(val, dict): + for subk, subv in val.items(): + out.setdefault(subk, []) + if isinstance(subv, (list, tuple)): + out[subk].extend(subv) + else: + out[subk].append(subv) + # if the value is a list, record items under '' + elif isinstance(val, (list, tuple)): + out[''].extend(val) + else: + # record simple scalar values under their field name + out.setdefault(top_key, []).append(val) + return out + + +def _default_splitter(s: Any) -> List[Any]: + """ + Default splitter: semicolon-split for strings, wrap others in a single-item list. + """ + if isinstance(s, str): + return s.split(";") + return [s] + + +class UniquenessDFBlock(AnimalBlock): + """ + Summarize uniqueness of each column in a DataFrame. + + Needs: 'df' + Provides: 'uniqueness_df' + + Parameters + ---------- + splitters : Optional[Dict[str, Callable[[Any], Union[Sequence[Any], Dict[str, List[Any]]]]]] + Column-specific split functions. Overrides defaults. + A splitter may return a flat sequence or a dict of sub-sequences. + exclude_cols : Sequence[str], optional + Column names to skip entirely (defaults to ['title','abstract','num_citations','num_references']). + """ + DEFAULT_SPLITTERS: Dict[str, Callable[[Any], Any]] = { + # semicolon-delimited lists + "authors": lambda s: s.split(";") if isinstance(s, str) else [], + "author_ids": lambda s: s.split(";") if isinstance(s, str) else [], + "s2_authors": lambda s: s.split(";") if isinstance(s, str) else [], + "s2_author_ids": lambda s: s.split(";") if isinstance(s, str) else [], + "PACs": lambda s: s.split(";") if isinstance(s, str) else [], + "subject_areas": lambda s: s.split(";") if isinstance(s, str) else [], + "citations": lambda s: s.split(";") if isinstance(s, str) else [], + "references": lambda s: s.split(";") if isinstance(s, str) else [], + # JSON-like dicts: use generic nested splitter + "affiliations": _nested_dict_splitter, + "funding": _nested_dict_splitter, + } + CANONICAL_NEEDS = ('df',) + + def __init__( + self, + splitters: Optional[Dict[str, Callable[[Any], Any]]] = None, + exclude_cols: Sequence[str] = ('title', 'abstract', 'num_citations', 'num_references'), + needs=CANONICAL_NEEDS, + provides: Sequence[str] = ('uniqueness_df',), + **kwargs: Any, + ) -> None: + super().__init__(needs=needs, provides=provides, **kwargs) + + self.splitters: Dict[str, Callable[[Any], Any]] = { + **self.DEFAULT_SPLITTERS + } + if splitters: + self.splitters.update(splitters) + + self.exclude_cols = set(exclude_cols) + + def run(self, bundle: DataBundle) -> DataBundle: + df: pd.DataFrame = bundle['df'] + result_rows: List[Dict[str, Any]] = [] + + for col in df.columns: + if col in self.exclude_cols: + continue + series = df[col].dropna() + + splitter = self.splitters.get(col, _default_splitter) + + # Prepare a dict of sets to collect unique items + col_sets: Dict[str, set] = {} + + for cell in series: + try: + out = splitter(cell) + except Exception: + out = [] + + if isinstance(out, dict): + for subkey, items in out.items(): + target = subkey or '' + col_sets.setdefault(target, set()) + for item in items: + if pd.isna(item): + continue + col_sets[target].add(item) + else: + col_sets.setdefault('', set()) + for item in out: + if pd.isna(item): + continue + col_sets[''].add(item) + + # Build row entries for each subkey + for subkey, items in col_sets.items(): + subcol = f"{col}.{subkey}" if subkey else col + count = len(items) + # values_str = ';'.join(map(str, sorted(items))) + # sort heterogenous items by their string representation + sorted_items = sorted(items, key=lambda x: str(x)) + values_str = ';'.join(map(str, sorted_items)) + result_rows.append({'col': subcol, 'count': count, 'values': values_str}) + + uniqueness_df = pd.DataFrame(result_rows, columns=[ 'count', 'col','values']) + + + if SAVE_DIR_BUNDLE_KEY in bundle: + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(exist_ok=True, parents=True) + final_csv = out_dir / f"{self.tag}.csv" + uniqueness_df.to_csv(final_csv, index=False, encoding="utf-8-sig") + self.register_checkpoint(self.provides[0], final_csv) + + bundle[f"{self.tag}.{self.provides[0]}"] = uniqueness_df + + return bundle diff --git a/TELF/pipeline/blocks/vulture_clean_block.py b/TELF/pipeline/blocks/vulture_clean_block.py new file mode 100644 index 00000000..0f3594cd --- /dev/null +++ b/TELF/pipeline/blocks/vulture_clean_block.py @@ -0,0 +1,159 @@ +# blocks/vulture_clean_block.py +from __future__ import annotations + +from pathlib import Path +from typing import Dict, Sequence, Any + +from ...pre_processing.Vulture import Vulture +from ...pre_processing.Vulture.modules import ( + RemoveNonEnglishCleaner, + SimpleCleaner, + LemmatizeCleaner, + SubstitutionCleaner, +) +from ...pre_processing.Vulture.default_stop_words import STOP_WORDS +from ...pre_processing.Vulture.default_stop_phrases import STOP_PHRASES + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +# from ...helpers.terms import resolve_substitution_conflicts +import spacy + +class VultureCleanBlock(AnimalBlock): + """ + Text-clean a DataFrame with **Vulture**. + + ───────────────────────────────────────────────────────────── + always needs : ('df',) – takes the *latest* DataFrame + needs 'substitutions' only when use_substitutions=True + provides : ('df', 'vulture_steps') – writes a cleaned copy + tag : 'VultureClean' (namespace for its outputs) + """ + CANONICAL_NEEDS = ("df",) + + def __init__( + self, + *, + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ("df", "vulture_steps",), + checkpoint_keys=("df",), + use_substitutions:bool = False, + conditional_needs: Sequence[tuple[str, Any]] = (), # none today + tag: str = "VultureClean", + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + **kw, + ) -> None: + + default_init = { + "n_jobs": -1, + "verbose": 10, + 'parallel_backend':'threading' + } + self.use_substitutions = use_substitutions + + + #nlp = spacy.load("en_core_web_sm") + #for pipe in ["tok2vec","transformer","parser","ner","attribute_ruler","tagger"]: + # if pipe in nlp.pipe_names: + # nlp.remove_pipe(pipe) + + # now only ['lemmatizer'] (and the implicit tokenizer) remain + lemmatize_cleaner = LemmatizeCleaner("spacy") + #lemmatize_cleaner.backend = nlp + + self.steps = [ + RemoveNonEnglishCleaner(ascii_ratio=0.9, stopwords_ratio=0.25), + SimpleCleaner( + stop_words=STOP_WORDS, + stop_phrases=STOP_PHRASES, + order=[ + "standardize_hyphens", + "isolate_frozen", + "remove_copyright_statement", + "remove_stop_phrases", + "make_lower_case", + "remove_formulas", + "normalize", + "remove_next_line", + "remove_email", + "remove_()", + "remove_[]", + "remove_special_characters", + "remove_nonASCII_boundary", + "remove_nonASCII", + "remove_tags", + "remove_stop_words", + "remove_standalone_numbers", + "remove_extra_whitespace", + "min_characters", + "remove_numbers", + "remove_alphanumeric", + "remove_roman_numerals", + ], + ), + lemmatize_cleaner, + ] + default_call = { + "columns": ["title", "abstract"], + "append_to_original_df": True, + "concat_cleaned_cols": True, + "steps": self.steps + } + + conds = list(conditional_needs or []) + if self.use_substitutions: + conds.append(("substitutions", lambda _b, _s: True)) + + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conds, + checkpoint_keys=checkpoint_keys, # ← add this line + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + **kw, + ) + + # ------------------------------------------------------------------ # + # internal helper # + # ------------------------------------------------------------------ # + def __rebuild_steps(self, bundle: DataBundle): + """Reconstruct the full steps list, including substitutions if enabled.""" + if self.use_substitutions: + subs_map = self.load_path(bundle["substitutions"]) + initial = SubstitutionCleaner(subs_map, permute=True, lower=True, lemmatize=True) + final = SubstitutionCleaner(subs_map, permute=False, lower=False, lemmatize=True) + return [initial] + self.steps + [final] + else: + return self.steps + + # ------------------------------------------------------------------ # + # work # + # ------------------------------------------------------------------ # + def run(self, bundle: DataBundle) -> None: + df = self.load_path(bundle[self.needs[0]]) + call_cfg = dict(self.call_settings) + + steps = self.__rebuild_steps(bundle) + call_cfg["steps"] = steps + + vulture = Vulture(**self.init_settings) + cleaned_df = vulture.clean_dataframe(df, **call_cfg) + + if SAVE_DIR_BUNDLE_KEY in bundle: + out_dir = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + out_dir.mkdir(exist_ok=True, parents=True) + final_csv = out_dir / "clean_df.csv" + cleaned_df.to_csv(final_csv, index=False, encoding="utf-8-sig") + self.register_checkpoint(self.provides[0], final_csv) + + bundle[f"{self.tag}.{self.provides[0]}"] = cleaned_df + bundle[f"{self.tag}.{self.provides[1]}"] = steps + + def _after_checkpoint_skip(self, bundle: DataBundle) -> None: + """Re-insert fast-to-make steps after checkpoint reload.""" + steps = self.__rebuild_steps(bundle) + bundle[f"{self.tag}.{self.provides[1]}"] = steps \ No newline at end of file diff --git a/TELF/pipeline/blocks/wolf_block.py b/TELF/pipeline/blocks/wolf_block.py new file mode 100644 index 00000000..46af8ee5 --- /dev/null +++ b/TELF/pipeline/blocks/wolf_block.py @@ -0,0 +1,193 @@ +# wolf_block.py + +from __future__ import annotations +from typing import Dict, Sequence, Any, Tuple +import os +from pathlib import Path + +import numpy as np +import networkx as nx +from tqdm import tqdm + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY + +from ...post_processing import Wolf +from ...post_processing.Wolf.utils import create_attributes +from ...post_processing.Wolf.plots import component_wordclouds, save_components + +from ...helpers.file_system import check_path +from ...helpers.figures import plot_authors_graph +from ...helpers.frames import apply_alpha +from ...helpers.maps import get_id_to_name + +from .beaver_codependency_matrix_block import CodependencyMatrixBlock + + +class WolfBlock(AnimalBlock): + """ + needs: ['df', 'map'] + provides: ['graph'] + Automatically checkpoints 'graph' to disk as 'graph.gpickle'. + """ + + CANONICAL_NEEDS = ('df', 'map') + WOLF_STATS = ['page_rank', 'hubs_authorities', 'betweenness_centrality'] + + category_map = { + "co-author": { + "col": "slic_author_ids", + "name_col": "slic_authors", + "png": "all_co-authors.png", + "ranks": "co-author_rankings.csv", + "html": "co-authors.html", + }, + "co-affiliation": { + "col": "affiliation_ids", + "name_col": "affiliation_names", + "png": "all_co-affiliations.png", + "ranks": "co-affiliation_rankings.csv", + "html": "co-affiliations.html", + }, + "co-country": { + "col": "countries", + "name_col": "countries", + "png": "all_co-countries.png", + "ranks": "co-country_rankings.csv", + "html": "co-countries.html", + }, + } + + def __init__( + self, + *, + category: str = 'co-author', + needs: Sequence[str] = CANONICAL_NEEDS, + provides: Sequence[str] = ('graph',), + tag: str = "Wolf", + conditional_needs: Sequence[Tuple[str, Any]] = (), + init_settings: Dict[str, Any] = None, + call_settings: Dict[str, Any] = None, + verbose: bool = True, + ) -> None: + if category not in self.category_map: + raise ValueError(f"Unknown category {category!r}") + + self.category = category + + # allow multiple WolfBlock instances without key collision + if provides == ('graph',): + provides = (f"graph_{self.category}",) + + default_init = {"verbose": True} + default_call = {} + + # By default, checkpoint_keys is None → AnimalBlock will use self.provides + super().__init__( + needs=needs, + provides=provides, + conditional_needs=conditional_needs, + tag=tag, + init_settings={**default_init, **(init_settings or {})}, + call_settings={**default_call, **(call_settings or {})}, + verbose=verbose, + ) + + def run(self, bundle: DataBundle) -> None: + # ─── 1) Load the DataFrame & map ────────────────────────────────────── + df = self.load_path(bundle[self.needs[0]]) + orca_map = bundle[self.needs[1]] + + OUTPUT_DIR = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + output_dir = Path(check_path(os.path.join(OUTPUT_DIR, self.category))) + + # ─── 2) Build the codependency matrix ──────────────────────────────── + codep = CodependencyMatrixBlock( + col=self.category_map[self.category]['col'] + ) + sub_bundle = DataBundle({ + 'df': df, + SAVE_DIR_BUNDLE_KEY: OUTPUT_DIR + }) + codep(sub_bundle) + X, node_ids = ( + sub_bundle[codep.provides[0]], + sub_bundle[codep.provides[1]] + ) + + # ─── 3) Prepare node attributes ────────────────────────────────────── + wolf = Wolf(**self.init_settings) + wolf.node_ids = node_ids + + if self.category == "co-author": + wolf.attributes = create_attributes(orca_map, attribute_names=[]) + + + elif self.category == "co-affiliation": + name_col = self.category_map[self.category]['name_col'] + id_col = self.category_map[self.category]['col'] + mapping = get_id_to_name(df, name_col, id_col) + wolf.attributes = {k: {'name': v} for k, v in mapping.items()} + + # ─── 4) Create & annotate the graph ─────────────────────────────────── + graph = wolf.create_graph(X, use_weighted_value=True) + for stat in tqdm(self.WOLF_STATS): + graph.get_stat(stat) + + # ─── 5) Output rankings CSV ────────────────────────────────────────── + stats_df = graph.output_stats() + numeric = stats_df.select_dtypes(include=[np.number]).columns + stats_df[numeric] = stats_df[numeric].map(apply_alpha) + stats_df = stats_df \ + .sort_values(by=next(iter(self.WOLF_STATS)), ascending=False) \ + .reset_index(drop=True) + + stats_df.to_csv( + output_dir / self.category_map[self.category]['ranks'], + index=False, + encoding="utf-8-sig" + ) + + # ─── 6) Save full-network plot & HTML ──────────────────────────────── + graph.visualize( + font_color = 'black', + node_color = '#edede9', + node_size = 100, + highlight_nodes = [], + font_size = 4, + edge_width = 0.08, + figsize = (8, 8), + save_path = str(output_dir / self.category_map[self.category]['png']) + ) + + fig = plot_authors_graph( + df = df, + id_col = self.category_map[self.category]['col'], + name_col = self.category_map[self.category]['name_col'], + ) + fig.write_html(str(output_dir / self.category_map[self.category]['html'])) + + # ─── 7) Component subplots & word-clouds ───────────────────────────── + save_components( + df = df, + ranking_df = stats_df, + g = graph, + col = self.category_map[self.category]['col'], + results_dir = str(output_dir), + ) + component_wordclouds( + df = df, + g = graph, + col = self.category_map[self.category]['col'], + results_dir = str(output_dir), + ) + + # ─── 8) Checkpoint the graph ───────────────────────────────────────── + graph_path = output_dir / "graph.gpickle" + graph_path.parent.mkdir(parents=True, exist_ok=True) + self.save_path(graph, graph_path) + + # Tell AnimalBlock to record this file under the key "graph" + self.register_checkpoint(self.provides[0], graph_path) + # Finally, put the graph into the bundle under your namespaced key + bundle[f"{self.tag}.{self.provides[0]}"] = graph \ No newline at end of file diff --git a/TELF/pipeline/blocks/wordcloud_block.py b/TELF/pipeline/blocks/wordcloud_block.py new file mode 100644 index 00000000..6867ff4e --- /dev/null +++ b/TELF/pipeline/blocks/wordcloud_block.py @@ -0,0 +1,57 @@ +from pathlib import Path +from typing import Any, Dict + +from .base_block import AnimalBlock +from .data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY +from ...helpers.figures import create_wordcloud_from_df + +class WordCloudBlock(AnimalBlock): + CANONICAL_NEEDS = ('df', ) + + def __init__( + self, + needs=CANONICAL_NEEDS, + provides=("wordcloud",), + tag='Wordcloud', + *, + init_settings: Dict[str, Any] = None, + call_settings: Dict[str, Any] = None, + **kw + ): + default_init = {} + default_call = { + 'col': 'clean_title_abstract', + 'n': 30, + 'save_path': None, + 'figsize': (6, 6), + } + + super().__init__( + needs=needs, + provides=provides, + tag=tag, + init_settings=self._merge(default_init, init_settings), + call_settings=self._merge(default_call, call_settings), + **kw, + ) + + def run(self, bundle: DataBundle) -> None: + # 1) Load the DataFrame and drop any rows where the target column is NaN + df = ( + self.load_path(bundle[self.needs[0]]) + .dropna(subset=[self.call_settings['col']]) + ) + + # 2) Generate the wordcloud (first positional arg is df) + create_wordcloud_from_df(df, **self.call_settings) + + # 3) Decide on the output PNG path + save_path = self.call_settings['save_path'] + if save_path is not None: + out_path = Path(save_path) + else: + out_path = Path(bundle[SAVE_DIR_BUNDLE_KEY]) / self.tag + + # 4) “Load” that path (or simply return it) and record in the bundle + png = self.load_path(out_path) + bundle[f"{self.tag}.{self.provides[0]}"] = png diff --git a/TELF/pipeline/dir_loop_block.py b/TELF/pipeline/dir_loop_block.py new file mode 100644 index 00000000..9fad224d --- /dev/null +++ b/TELF/pipeline/dir_loop_block.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, List, Sequence, Tuple +from concurrent.futures import ThreadPoolExecutor, as_completed + +from .blocks.base_block import AnimalBlock +from .blocks.data_bundle import ( + DataBundle, + SAVE_DIR_BUNDLE_KEY, + SOURCE_DIR_BUNDLE_KEY, + DIR_LIST_BUNDLE_KEY, + RESULTS_DEFAULT, +) +from .block_manager import BlockManager + +import traceback + + +def _clone_databundle(bundle: DataBundle) -> DataBundle: + """ + Clone a DataBundle by copying its internal _store dict, + which includes each base-key bucket and its '_latest' pointer. + """ + clone = DataBundle() + clone.__dict__['_store'] = { + base: bucket.copy() for base, bucket in bundle._store.items() + } + return clone + + +class DirectoryLoopBlock(AnimalBlock): + """ + For each path in bundle[DIR_LIST_BUNDLE_KEY]: + - clone the incoming DataBundle via _clone_databundle + - set SOURCE_DIR_BUNDLE_KEY and 'dir' + - optionally override SAVE_DIR_BUNDLE_KEY to SOURCE_DIR_BUNDLE_KEY/RESULTS_DEFAULT + - run subblocks in parallel, isolating logs per run + Collects each sub-bundle's .as_dict() under bundle[tag.RESULTS_DEFAULT]. + """ + CANONICAL_NEEDS = (DIR_LIST_BUNDLE_KEY,) + + def __init__( + self, + subblocks: List[AnimalBlock], + needs: Tuple[str, ...] = CANONICAL_NEEDS, + provides: Tuple[str, ...] = (RESULTS_DEFAULT,), + tag: str = 'DirLoop', + conditional_needs: Sequence[Tuple[str, Any]] = (), + capture_output: str = 'file', + use_source_dir: bool = True, + *, + init_settings: Dict[str, Any] | None = None, + call_settings: Dict[str, Any] | None = None, + max_workers: int | None = None, + verbose: bool = True, + force_checkpoint: bool | None = None, + **kwargs, + ) -> None: + self.use_source_dir = use_source_dir + self.force_checkpoint = force_checkpoint + super().__init__( + needs=needs, + provides=provides, + init_settings=self._merge({}, init_settings), + call_settings=self._merge({}, call_settings), + conditional_needs=conditional_needs, + tag=tag, + verbose=verbose, + **kwargs, + ) + self.capture_output = capture_output + self.subblocks = subblocks + self.max_workers = max_workers + + def run(self, bundle: DataBundle) -> DataBundle: + dirs = list(bundle[self.needs[0]]) + total = len(dirs) + results: List[Dict[str, Any]] = [] + base_bundle = bundle + + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + future_to_path = { + executor.submit(self._process_one, base_bundle, path): path + for path in dirs + } + + completed = 0 + for future in as_completed(future_to_path): + path = future_to_path[future] + try: + sub_res = future.result() + except Exception: + if self.verbose: + print(f"❌ {path!r} raised an exception:") + traceback.print_exc() + else: + results.append(sub_res) + completed += 1 + if self.verbose: + print(f"✅ [{completed}/{total}] Completed processing {path!r}") + + bundle[f"{self.tag}.{self.provides[0]}"] = results + return bundle + + def _process_one(self, base_bundle: DataBundle, path: Path) -> Dict[str, Any]: + try: + sub = _clone_databundle(base_bundle) + sub[SOURCE_DIR_BUNDLE_KEY] = Path(path) + sub['dir'] = Path(path) + + if self.use_source_dir: + sub[SAVE_DIR_BUNDLE_KEY] = sub[SOURCE_DIR_BUNDLE_KEY] / RESULTS_DEFAULT + + if self.verbose: + print(f"\n▶ Starting {path!r}") + print(f" Output directory: {sub[SAVE_DIR_BUNDLE_KEY]!r}") + + mgr = BlockManager( + blocks=self.subblocks, + databundle=sub, + verbose=self.verbose, + capture_output=self.capture_output, + force_checkpoint=self.force_checkpoint, + ) + result_bundle = mgr() + return result_bundle.as_dict() + + except Exception: + print(f"⚠️ Error processing {path!r} in _process_one:") + traceback.print_exc() + raise diff --git a/TELF/pipeline/loop_block.py b/TELF/pipeline/loop_block.py new file mode 100644 index 00000000..bb7e8f15 --- /dev/null +++ b/TELF/pipeline/loop_block.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, List, Sequence +from concurrent.futures import ThreadPoolExecutor, as_completed +import traceback + +from .blocks.base_block import AnimalBlock +from .blocks.data_bundle import DataBundle, SAVE_DIR_BUNDLE_KEY, RESULTS_DEFAULT +from .block_manager import BlockManager + + +def _clone_databundle(bundle: DataBundle) -> DataBundle: + """ + Shallow-clone a DataBundle, preserving each base-key bucket and its + '_latest' pointer. + """ + clone = DataBundle() + clone.__dict__['_store'] = { + base: bucket.copy() for base, bucket in bundle._store.items() + } + return clone + + +class RepeatLoopBlock(AnimalBlock): + """ + Run a list of `subblocks` `n_iter` times. + Its .needs are taken from the *first* subblock, and its .provides is only RESULTS_DEFAULT. + + Ensures all iterations write into a single `/RepeatLoop/iter_xx` tree, + without nesting off of previous iterations. After completion, restores + the original SAVE_DIR so subsequent blocks write to the correct location. + + If `stop_on_error` is True (default), the loop aborts on the first iteration error, + returning only the successful results so far. + """ + + def __init__( + self, + subblocks: List[AnimalBlock], + *, + n_iter: int, + clone: bool = False, + parallel: bool = False, + max_workers: int | None = None, + redirect_save_dir: bool = True, + tag: str = "RepeatLoop", + capture_output: str = "file", + verbose: bool = True, + stop_on_error: bool = True, + force_checkpoint: bool | None = None, + **kwargs: Any, + ) -> None: + self.subblocks = subblocks + self.n_iter = int(n_iter) + self.clone = clone + self.parallel = parallel + self.max_workers = max_workers + self.redirect_save_dir = redirect_save_dir + self.capture_output = capture_output + self.verbose = verbose + self.stop_on_error = stop_on_error + self.force_checkpoint = force_checkpoint + + # take needs from the first subblock only: + first = subblocks[0] + needs = tuple(first.needs) + tuple(key for key, _ in first.conditional_needs) + + # this block itself only provides the collected results list + provides = (RESULTS_DEFAULT,) + + super().__init__( + needs=needs, + provides=provides, + tag=tag, + verbose=verbose, + capture_output=capture_output, + **kwargs, + ) + + def run(self, bundle: DataBundle) -> DataBundle: + results: List[Dict[str, Any]] = [] + total = self.n_iter + + # Preserve the original save directory + orig_save = bundle.get(SAVE_DIR_BUNDLE_KEY) + + # Determine base directory for all iterations + base_save = Path(orig_save) if orig_save is not None else Path.cwd() / "results" + loop_root = base_save / self.tag if self.redirect_save_dir else None + + if self.parallel: + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + futures = { + executor.submit(self._one_pass, bundle, i, loop_root): i + for i in range(total) + } + completed = 0 + for fut in as_completed(futures): + i = futures[fut] + try: + res = fut.result() + except Exception: + if self.verbose: + print(f"❌ iter={i} raised exception:") + traceback.print_exc() + if self.stop_on_error: + for pending in futures: + if not pending.done(): + pending.cancel() + break + else: + continue + else: + results.append(res) + completed += 1 + if self.verbose: + print(f"✅ [{completed}/{total}] finished iter={i}") + else: + for i in range(total): + if self.verbose: + print(f"\n▶ [{i+1}/{total}] starting iter={i}") + try: + res = self._one_pass(bundle, i, loop_root) + except Exception: + if self.stop_on_error: + if self.verbose: + print(f"⏹ stopping after iter={i} due to error.") + break + else: + continue + else: + results.append(res) + if self.verbose: + print(f"✓ [{i+1}/{total}] finished iter={i}") + + # store whatever successful results we have + bundle[f"{self.tag}.{RESULTS_DEFAULT}"] = results + + # Restore original save directory for subsequent blocks + if orig_save is not None: + bundle[SAVE_DIR_BUNDLE_KEY] = orig_save + + return bundle + + def _one_pass(self, base_bundle: DataBundle, idx: int, loop_root: Path | None) -> Dict[str, Any]: + try: + sub = _clone_databundle(base_bundle) if self.clone else base_bundle + sub['iter'] = idx + + if loop_root is not None: + sub[SAVE_DIR_BUNDLE_KEY] = loop_root / f"iter_{idx:02d}" + + mgr = BlockManager( + blocks=self.subblocks, + databundle=sub, + verbose=self.verbose, + capture_output=self.capture_output, + force_checkpoint=self.force_checkpoint, + ) + return mgr().as_dict() + + except Exception: + print(f"⚠️ Error in RepeatLoopBlock (iter={idx}):") + traceback.print_exc() + raise diff --git a/TELF/post_processing/ArcticFox/__init__.py b/TELF/post_processing/ArcticFox/__init__.py index 0ac91a84..7d9e2d0a 100644 --- a/TELF/post_processing/ArcticFox/__init__.py +++ b/TELF/post_processing/ArcticFox/__init__.py @@ -2,3 +2,5 @@ from .helpers.intial_post_process import HNMFkPostProcessor from .helpers.post_statistics import HNMFkStatsGenerator from .arcticfox import ArcticFox +from .cluster_analyzer import ClusteringAnalyzer +from .label_analyzer import LabelAnalyzer \ No newline at end of file diff --git a/TELF/post_processing/ArcticFox/cluster_analyzer.py b/TELF/post_processing/ArcticFox/cluster_analyzer.py new file mode 100644 index 00000000..a76f7af3 --- /dev/null +++ b/TELF/post_processing/ArcticFox/cluster_analyzer.py @@ -0,0 +1,307 @@ +import os +from pathlib import Path +from collections import Counter +from typing import List, Optional + +import numpy as np +import pandas as pd + +from ...helpers.stats import H_clustering, top_words +from ...helpers.figures import create_wordcloud, plot_H_clustering +from ...helpers.file_system import check_path +from ...post_processing.Fox.post_process_functions import get_core_map, H_cluster_argmax, best_n_papers +from ...pre_processing.Vulture.tokens_analysis.top_words import get_top_words + + +class ClusteringAnalyzer: + # ────────────────────────────────────────────────────────────────── + def __init__(self, **kwargs): + defaults = dict( + top_n_words=50, + out_dir='./post_result', + archive_subdir='archive', + default_clean_col='clean_title_abstract', + wordcloud_size=(800, 800), + max_font_size=80, + contour_width=1, + col_year='year', + col_type='type', + col_cluster='cluster', + col_cluster_coords='cluster_coordinates', + col_similarity='similarity_to_cluster_centroid', + table_filename='table_H-clustering.csv', + cluster_doc_map_filename='cluster_documents_map.txt', + top_words_filename='top_words.csv', + probs_filename='probabilities_top_words.csv', + clusters_info_filename='clusters_information.csv', + documents_info_filename='documents_information.csv' + ) + self.__dict__.update({**defaults, **kwargs}) + + # ────────────────────────────────────────────────────────────────── + # public entry point (now supports hierarchical HNMFk as well) + # ────────────────────────────────────────────────────────────────── + def analyze( + self, + df, + W: Optional[np.ndarray] = None, + H: Optional[np.ndarray] = None, + vocab: Optional[np.ndarray] = None, + cluster_col: Optional[str] = None, + *, + hnmfk_model=None, + clean_cols_name=None, + process_parents=False, + skip_completed=True, + ): + """ + Main dispatcher. + + • W/H/vocab present → single NMF-k matrix + • hnmfk_model + vocab → hierarchical (runs each node) + • cluster_col provided → pre-existing labels + • else → pass-through (everything in one cluster) + """ + self.archive_dir = self._ensure_dirs() + df_copy = df.copy() + self.default_clean_col = clean_cols_name or self.default_clean_col + + if hnmfk_model is not None and vocab is not None: + return self.analyze_hnmfk( + hnmfk_model, vocab, df_copy, + clean_cols_name=clean_cols_name or self.default_clean_col, + process_parents=process_parents, + skip_completed=skip_completed, + ) + + if W is not None and H is not None and vocab is not None: + return self._analyze_factor_model(df_copy, W, H, vocab) + elif cluster_col and cluster_col in df_copy: + return self._analyze_label_based(df_copy, cluster_col) + else: + return self._analyze_pass_through(df_copy) + + # ────────────────────────────────────────────────────────────────── + # hierarchical HNMF-k processor + # ────────────────────────────────────────────────────────────────── + def analyze_hnmfk( + self, + hnmfk_model, + vocab, + data_df, + *, + clean_cols_name, + process_parents=False, + skip_completed=True, + ) -> List[Path]: + """ + Run post-processing on every leaf node (or all nodes if + `process_parents=True`) of a fitted HNMFk model. + + Returns a list of CSV paths created. + """ + vocab = np.asarray(vocab) + out_csvs: List[Path] = [] + + for node in hnmfk_model.traverse_nodes(): + if not (node["leaf"] or process_parents): + continue + + # ── gather W / H for this node ──────────────────────────── + W = node.get("W") + H = node.get("H") + if W is None and H is None: # signature-only node + W = node["signature"].reshape(-1, 1) + H = node["probabilities"].reshape(1, -1) + + node_dir = Path(node["node_save_path"]).resolve().parent + csv_out = node_dir / f"cluster_for_k={W.shape[1]}.csv" + if skip_completed and csv_out.exists(): + out_csvs.append(csv_out) + continue + + idxs = list(node["original_indices"]) + node_df = data_df.iloc[idxs].reset_index(drop=True) + + # redirect outputs into the node’s folder + prev_out_dir = self.out_dir + self.out_dir = str(node_dir) + self.archive_dir = self._ensure_dirs() + + self._analyze_factor_model(node_df, W, H, vocab) + + # restore global out_dir + self.out_dir = prev_out_dir + out_csvs.append(csv_out) + + return out_csvs + + # ────────────────────────────────────────────────────────────────── + # helpers + # ────────────────────────────────────────────────────────────────── + def _ensure_dirs(self): + archive_dir = os.path.join(self.out_dir, self.archive_subdir) + os.makedirs(archive_dir, exist_ok=True) + return archive_dir + + def _save_cluster_info(self, labels, table_df): + table_df.to_csv(os.path.join(self.out_dir, self.table_filename), index=False) + np.savetxt(os.path.join(self.archive_dir, self.cluster_doc_map_filename), + labels, fmt='%d') + + def _save_cluster_text_outputs(self, cluster_id, df, vocab=None, W=None): + save_dir = check_path(os.path.join(self.out_dir, str(cluster_id))) + # docs = df[self.default_clean_col].to_dict() + docs = ( + df[self.default_clean_col] + .fillna("") # replace NaN with empty string + .astype(str) # ensure everything is a str + .to_dict() + ) + + for n in (1, 2): + bow = get_top_words(docs, top_n=100, n_gram=n, verbose=True) + suffix = 'unigrams' if n == 1 else 'bigrams' + bow.to_csv(os.path.join(save_dir, f'{cluster_id}_bow_{suffix}.csv'), index=False) + + if vocab is None or W is None: # fallback: simple term-freq + tokens = [tok for doc in docs.values() for tok in doc.split()] + freq = Counter(tokens) + vocab, W = list(freq.keys()), np.array(list(freq.values())).reshape(-1, 1) + + create_wordcloud( + W=W, + vocab=vocab, + top_n=self.top_n_words, + path=save_dir, + max_words=self.top_n_words, + mask=np.zeros(self.wordcloud_size), + background_color='black', + max_font_size=self.max_font_size, + contour_width=self.contour_width + ) + + def _rank_docs(self, df): + for col in (self.col_similarity, self.col_year, 'citations', 'references'): + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors='coerce') + return df.nlargest(self.top_n_words, col) + + for col in ('author_ids', 'authors'): + if col in df.columns: + df = df.assign(author_count=df[col].apply( + lambda x: len(str(x).split(';')) if pd.notna(x) else 0)) + return df.nlargest(self.top_n_words, 'author_count') + + return df.sample(min(len(df), self.top_n_words), random_state=42) + + def _save_best_docs(self, df): + for cid in df[self.col_cluster].dropna().unique(): + cdf = df[df[self.col_cluster] == cid].copy() + best_df = self._rank_docs(cdf) + out_dir = os.path.join(self.out_dir, str(int(cid))) + best_df.to_csv(os.path.join(out_dir, + f'best_{self.top_n_words}_docs_in_{cid}.csv'), + index=False) + + # ────────────────────────────────────────────────────────────────── + # core analysis modes + # ────────────────────────────────────────────────────────────────── + def _analyze_factor_model(self, df, W, H, vocab): + labels, counts, table_df = H_cluster_argmax(H) + self._save_cluster_info(labels, table_df) + + vocab_arr = np.array(vocab) + words, probs = top_words(W, vocab_arr, self.top_n_words) + pd.DataFrame(words).to_csv(os.path.join(self.out_dir, + self.top_words_filename), + index=False) + pd.DataFrame(probs).to_csv(os.path.join(self.out_dir, + self.probs_filename), + index=False) + + create_wordcloud( + W=W, + vocab=vocab_arr, + top_n=self.top_n_words, + path=self.out_dir, + max_words=self.top_n_words, + mask=np.zeros(self.wordcloud_size), + background_color='black', + max_font_size=self.max_font_size, + contour_width=self.contour_width, + grid_dimension=4 + ) + + clusters_info, docs_info = H_clustering(H, verbose=True) + pd.DataFrame(clusters_info).T.to_csv(os.path.join( + self.archive_dir, self.clusters_info_filename), index=False) + df_docs = pd.DataFrame(docs_info).T + df_docs.to_csv(os.path.join( + self.archive_dir, self.documents_info_filename), index=False) + + df[self.col_cluster] = df_docs['cluster'] + if 'cluster_coordinates' in df_docs: + df[self.col_cluster_coords] = df_docs['cluster_coordinates'] + df[self.col_similarity] = df_docs['similarity_to_cluster_centroid'] + + if self.col_year in df: + df[self.col_year] = (df[self.col_year].fillna(-1) + .astype(int).replace(-1, np.nan)) + if self.col_type in df: + core_map = get_core_map(df) + total_core = (df[self.col_type] == 0).sum() + if total_core: + table_df['core_count'] = table_df['cluster'].map(core_map) + table_df['core_percentage'] = table_df['cluster'].map( + lambda c: round(100 * core_map.get(c, 0) / total_core, 2)) + table_df.to_csv(os.path.join(self.out_dir, + self.table_filename), index=False) + + out_csv = f'cluster_for_k={W.shape[1]}.csv' + df.to_csv(os.path.join(self.out_dir, out_csv), index=False) + + for cid in sorted(df[self.col_cluster].dropna().unique()): + self._save_cluster_text_outputs( + cid, + df[df[self.col_cluster] == cid], + vocab=vocab_arr, + W=W[:, int(cid)].reshape(-1, 1) + ) + plot_H_clustering( + H[int(cid), :].reshape(1, -1), + name=os.path.join(self.out_dir, str(cid), + f'centroids_H_clustering_{cid}.png') + ) + + self._save_best_docs(df) + return os.path.join(self.out_dir, out_csv) + + def _analyze_label_based(self, df, cluster_col): + df[self.col_cluster] = df[cluster_col] + labels = df[cluster_col].values + counts = pd.Series(labels).value_counts() + table_df = pd.DataFrame({"cluster": counts.index, + "count": counts.values}) + + self._save_cluster_info(labels, table_df) + for cid in counts.index: + self._save_cluster_text_outputs(cid, df[df[cluster_col] == cid]) + self._save_best_docs(df) + + out_csv = 'cluster_for_labels.csv' + df.to_csv(os.path.join(self.out_dir, out_csv), index=False) + return os.path.join(self.out_dir, out_csv) + + def _analyze_pass_through(self, df): + df[self.col_cluster] = 0 + labels = df[self.col_cluster].values + table_df = pd.DataFrame({"cluster": [0], "count": [len(df)]}) + + self._save_cluster_info(labels, table_df) + self._save_cluster_text_outputs(0, df) + self._save_best_docs(df) + + out_csv = 'clustered_pass_through.csv' + df.to_csv(os.path.join(self.out_dir, out_csv), index=False) + return os.path.join(self.out_dir, out_csv) diff --git a/TELF/post_processing/ArcticFox/helpers/local_labels.py b/TELF/post_processing/ArcticFox/helpers/local_labels.py index e65576c0..26726fdc 100644 --- a/TELF/post_processing/ArcticFox/helpers/local_labels.py +++ b/TELF/post_processing/ArcticFox/helpers/local_labels.py @@ -4,9 +4,11 @@ from sklearn.metrics import pairwise_distances from langchain_ollama import OllamaLLM -from ....helpers.embeddings import (compute_doc_embedding, produce_label, - compute_embeddings, closest_embedding_to_centroid, +from ....helpers.embeddings import (compute_doc_embedding, compute_embeddings, + closest_embedding_to_centroid, compute_centroids) +from ....helpers.llm_operations import produce_label +from ....helpers.llm_models import get_openai_llm class ClusterLabeler: @@ -122,7 +124,8 @@ def labels_from_models( if models.get('openai'): openai_api_key = api_keys.get('openai') for _ in range(number_of_labels): - candidate = produce_label(openai_api_key, input_words) + client = get_openai_llm(openai_api_key=openai_api_key) + candidate = produce_label(client=client, words=input_words, model="gpt-3.5-turbo-instruct") if self.validate_llm_output(candidate, criteria): all_labels.append(candidate) diff --git a/TELF/post_processing/ArcticFox/label_analyzer.py b/TELF/post_processing/ArcticFox/label_analyzer.py new file mode 100644 index 00000000..4982a7d8 --- /dev/null +++ b/TELF/post_processing/ArcticFox/label_analyzer.py @@ -0,0 +1,309 @@ +""" +LabelAnalyzer – self-contained, patched for your compute_embeddings signature +──────────────────────────────────────────────────────────────────────────── +Produces specific labels for: + 1) single string + 2) list of texts + 3) DataFrame one-row per text + 4) DataFrame grouped by cluster_col + +Supports both Ollama and OpenAI chat models. + +Dependencies +------------ +pip install "langchain-community>=0.2.0" "langchain-openai>=0.1.1" \ + pandas numpy scikit-learn tqdm +""" +from __future__ import annotations +import re +from collections import Counter +from typing import Any, Dict, Iterable, List, Mapping, Optional, Union + +import numpy as np +import pandas as pd +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.metrics import pairwise_distances +from tqdm import tqdm + +# ── LLM back-ends ─────────────────────────────────────────────────────────── +from langchain_community.chat_models import ChatOllama +from langchain_openai import ChatOpenAI +from langchain.schema import SystemMessage, HumanMessage +from langchain_ollama import OllamaLLM + +# ── YOUR embedding helpers ────────────────────────────────────────────────── +from ...helpers.embeddings import ( + compute_doc_embedding, + compute_embeddings, + closest_embedding_to_centroid, + compute_centroids, +) + +# ── misc globals ──────────────────────────────────────────────────────────── +TOKEN_RE = re.compile(r"[A-Za-z]{3,}") +GENERIC = { + "analysis","approach","application","model","models","method","methods", + "research","study","studies","data","system","systems","framework", + "technology","technologies","network","networks","learning" +} +def _tok(txt: str) -> List[str]: + return TOKEN_RE.findall(txt.lower()) + + +class LabelAnalyzer: + def __init__( + self, + embedding_model: str = "SCINCL", # kept for API compatibility + distance_metric: str = "cosine", + text_cols: Optional[List[str]] = None, + tfidf_top_k: int = 12, + include_bigrams: bool = True, + ): + self.embedding_model = embedding_model + self.distance_metric = distance_metric + self.text_cols = text_cols or ["title", "abstract"] + self.tfidf_top_k = tfidf_top_k + self.include_bigrams = include_bigrams + + @staticmethod + def _crit() -> Dict[str, Any]: + return { + "minimum words": 2, + "maximum words": 10, + "must_not_contain": ["\n", "\r", " ", "•", "-", ":"] + } + + def _clean(self, s: str) -> str: + first = s.splitlines()[0] + first = re.sub(r"^\s*([-\d]+[.)]|\u2022)\s*", "", first) + return re.sub(r"\s+", " ", first).strip() + + def _generic(self, s: str) -> bool: + toks = _tok(s) + return bool(toks) and all(t in GENERIC for t in toks) + + def _valid(self, s: str, crit: Dict[str,Any]) -> bool: + if self._generic(s): + return False + n = len(s.split()) + if not (crit["minimum words"] <= n <= crit["maximum words"]): + return False + if any(b in s for b in crit["must_not_contain"]): + return False + return True + + def _distinctive(self, df): + from collections import Counter + from sklearn.feature_extraction.text import TfidfVectorizer + + def simple_analyzer(text): + return text.lower().split() # replace with tokenizer/lemmatizer if needed + + docs = df["clean_text"].tolist() if "clean_text" in df.columns else df.iloc[:, 0].astype(str).tolist() + + # fallback for very small input + if len(df) < 2: + toks = simple_analyzer(" ".join(docs)) + return {0: [w for w, _ in Counter(toks).most_common(self.tfidf_top_k)]} + + try: + vec = TfidfVectorizer(analyzer=simple_analyzer, min_df=2) + X = vec.fit_transform(docs) + vocab = vec.get_feature_names_out() + cids = df["cluster"].values if "cluster" in df.columns else [0] * len(df) + return { + cid: [ + vocab[i] + for i in X[np.array(cids) == cid].sum(axis=0).A1.argsort()[::-1][:self.tfidf_top_k] + ] + for cid in sorted(set(cids)) + } + except ValueError: + toks = simple_analyzer(" ".join(docs)) + return {0: [w for w, _ in Counter(toks).most_common(self.tfidf_top_k)]} + + + def _prompt(self, freq, dist, titles, crit): + msg = ( + "### Task\nGenerate ONE short and descriptive **research topic label**.\n" + f"- Use {crit['minimum words']} to {crit['maximum words']} words\n" + "- Include a distinctive keyword\n" + "- DO NOT copy any of the titles\n" + "- Avoid punctuation, numbering, or full sentences\n\n" + f"**Frequent Keywords:** {', '.join(freq)}\n" + f"**Distinctive Keywords:** {', '.join(dist)}\n" + "**Representative Titles:**\n" + "\n".join(f"- {t}" for t in titles) + + "\nRespond ONLY with a concise label, not a sentence." + ) + return [ + SystemMessage("You are an expert at naming research topics."), + HumanMessage(msg) + ] + + def _cand_ollama(self, freq, dist, titles, crit, model, k): + chat = ChatOllama(model=model, base_url="http://localhost:11434", temperature=0.6) + raw = chat.invoke(self._prompt(freq, dist, titles, crit)).content + parts = [self._clean(p) for p in re.split(r"[\n,]+", raw) if p.strip()] + + # Remove any that match titles + title_set = set(t.lower().strip() for t in titles) + parts = [p for p in parts if p.lower().strip() not in title_set] + + return [p for p in parts if self._valid(p, crit)][:k] + + + def _cand_openai(self, freq, dist, titles, crit, model, key, k): + chat = ChatOpenAI(model=model, api_key=key, temperature=0.6) + raw = chat.invoke(self._prompt(freq, dist, titles, crit)).content + parts= [self._clean(p) for p in re.split(r"[\n,]+", raw) if p.strip()] + return [p for p in parts if self._valid(p, crit)][:k] + + def _centres(self, df: pd.DataFrame, gpu: bool) -> Dict[int, tuple]: + emb = compute_embeddings(df) + cents = compute_centroids(emb, df) # ← pass df here + return { + cid: closest_embedding_to_centroid(emb, c, metric=self.distance_metric) + for cid, c in cents.items() if c is not None + } + + + def _pick(self, cands: List[str], centre_vec: np.ndarray, gpu: bool) -> str: + dev = "cuda" if gpu else "cpu" + # ← FIXED: compute_doc_embedding takes only the text + embs = np.array([compute_doc_embedding(lbl) for lbl in cands]) + d = pairwise_distances(embs, centre_vec[None, :], + metric=self.distance_metric).ravel() + return cands[int(d.argmin())] + + def _to_clusters( + self, + data: Union[str, Iterable[str], Mapping[Any,str], pd.DataFrame], + *, + strat: str = "single", + col: str = "cluster", + ) -> Dict[Any, List[str]]: + if isinstance(data, str): + return {0: [data]} + if isinstance(data, Mapping): + return {k:[v] for k,v in data.items()} + if isinstance(data, (list, tuple, pd.Series)): + return ({i:[t] for i,t in enumerate(data)} if strat=="individual" + else {0:list(data)}) + if isinstance(data, pd.DataFrame): + joined = data[self.text_cols].astype(str).agg(" ".join, axis=1) + if strat=="column": + if col not in data.columns: + raise KeyError(f"cluster_col '{col}' missing") + grp = data.assign(_j=joined).groupby(col,sort=False)["_j"].apply(list) + return grp.to_dict() + if strat=="individual": + return {i:[t] for i,t in joined.items()} + return {0:list(joined)} + raise TypeError(f"Unsupported type {type(data)}") + + @staticmethod + def _top_words(clusters: Dict[Any,List[str]], n: int=20) -> pd.DataFrame: + def top(txts): + toks = (w.lower() for t in txts for w in t.split() + if w.isalpha() and len(w)>2) + return [w for w,_ in Counter(toks).most_common(n)] + return pd.DataFrame({cid:pd.Series(top(txts)) for cid,txts in clusters.items()}) + + def label_clusters( + self, + df: pd.DataFrame, + kw_df: pd.DataFrame, + *, + provider: str, + model: str, + api_key: Optional[str], + k: int, + crit: Dict[str,Any], + gpu: bool, + ) -> Dict[int,str]: + centres = self._centres(df, gpu) + dist_kw = self._distinctive(df) + out: Dict[int,str] = {} + + for cid in kw_df.columns: + freq_kw = kw_df[cid].dropna().tolist() + dist_kwl= dist_kw.get(int(cid),[])[:5] + titles = sorted( + df.loc[df["cluster"]==cid, self.text_cols] + .astype(str).agg(" ".join, axis=1), + key=len + )[:3] + + if provider=="ollama": + cands = self._cand_ollama(freq_kw, dist_kwl, titles, crit, model, k) + else: + cands = self._cand_openai(freq_kw, dist_kwl, titles, crit, model, api_key, k) + + if cid in centres and cands: + _, vec = centres[cid] + out[int(cid)] = self._pick(cands, vec, gpu) + + return out + + def label_texts( + self, + data: Union[str, Iterable[str], Mapping[Any,str], pd.DataFrame], + *, + provider: str = "ollama", + model_name: Optional[str] = None, + openai_api_key: Optional[str] = None, + cluster_strategy: str = "single", + cluster_col: str = "cluster", + top_n_words: int = 20, + num_candidates: int = 12, + use_gpu: bool = False, + ) -> Dict[int,str]: + provider = provider.lower() + if provider not in {"ollama","openai"}: + raise ValueError("provider must be 'ollama' or 'openai'") + if provider=="openai" and not openai_api_key: + raise ValueError("openai_api_key required for OpenAI") + + clusters = self._to_clusters(data, strat=cluster_strategy, col=cluster_col) + kw_df = self._top_words(clusters, top_n_words) + col0 = self.text_cols[0] + rows = [ + {"cluster":cid, col0:txt, **{c:txt for c in self.text_cols[1:]}} + for cid,txts in clusters.items() for txt in txts + ] + df = pd.DataFrame(rows) + + crit = self._crit() + model = model_name or ( + "llama3.2:3b-instruct-fp16" if provider=="ollama" else "gpt-3.5-turbo" + ) + return self.label_clusters( + df, kw_df, + provider=provider, + model=model, + api_key=openai_api_key, + k=num_candidates, + crit=crit, + gpu=use_gpu, + ) + + # convenience wrappers + def label_clusters_ollama(self, df, kw_df, **kw): + return self.label_clusters( + df, kw_df, provider="ollama", + model=kw.get("model_name","llama3.2:3b-instruct-fp16"), + api_key=None, + k=kw.get("num_candidates",12), + crit=self._crit(), + gpu=kw.get("use_gpu",False), + ) + + def label_clusters_openai(self, df, kw_df, openai_api_key, **kw): + return self.label_clusters( + df, kw_df, provider="openai", + model=kw.get("model_name","gpt-3.5-turbo"), + api_key=openai_api_key, + k=kw.get("num_candidates",12), + crit=self._crit(), + gpu=kw.get("use_gpu",False), + ) diff --git a/TELF/post_processing/Fox/__init__.py b/TELF/post_processing/Fox/__init__.py index 7ef39876..0161743b 100644 --- a/TELF/post_processing/Fox/__init__.py +++ b/TELF/post_processing/Fox/__init__.py @@ -2,8 +2,7 @@ from .openAI_summaries import (label_clusters_openAI) from .post_process_stats import (organize_top_words, get_cluster_stat_file, create_tensor, process_affiliations_coaffiliations) -from .post_process_functions import (get_core_map, H_cluster_argmax, - term_frequency, document_frequency, calculate_term_representations, +from .post_process_functions import (get_core_map, H_cluster_argmax, calculate_term_representations, best_n_papers, sme_attribution) from .visualizer import VisualizationManager diff --git a/TELF/post_processing/Fox/openAI_summaries.py b/TELF/post_processing/Fox/openAI_summaries.py index 70eb338f..b722bf4e 100644 --- a/TELF/post_processing/Fox/openAI_summaries.py +++ b/TELF/post_processing/Fox/openAI_summaries.py @@ -1,9 +1,11 @@ import numpy as np from tqdm import tqdm from sklearn.metrics import pairwise_distances -from ...helpers.embeddings import (compute_doc_embedding, produce_label, - compute_embeddings, closest_embedding_to_centroid, +from ...helpers.embeddings import (compute_doc_embedding, compute_embeddings, + closest_embedding_to_centroid, compute_centroids) +from ...helpers.llm_operations import produce_label +from ...helpers.llm_models import get_openai_llm MODEL = 'SCINCL' # either 'SCINCL' or 'SPECTER' DISTANCE_METRIC = 'cosine' # either 'cosine' or 'euclidean' @@ -20,7 +22,8 @@ def label_clusters(top_words_df, api_key, n_trials=1): words = top_words_df[col].to_list() for _ in range(n_trials): - label = produce_label(api_key, words) + client = get_openai_llm(openai_api_key=api_key) + label = produce_label(client=client, words=words, model="gpt-3.5-turbo-instruct") output[col_key].append(label) return output diff --git a/TELF/post_processing/Fox/post_process_functions.py b/TELF/post_processing/Fox/post_process_functions.py index 7c46cba3..409b0a4d 100644 --- a/TELF/post_processing/Fox/post_process_functions.py +++ b/TELF/post_processing/Fox/post_process_functions.py @@ -2,9 +2,9 @@ import warnings import numpy as np import pandas as pd -from sklearn.feature_extraction.text import TfidfVectorizer warnings.simplefilter(action='ignore', category=UserWarning) from ...helpers.file_system import check_path +from ...helpers.frames import calculate_term_representations # constants CONFIG_PATH = os.path.join("input", "config.json") @@ -22,63 +22,6 @@ def H_cluster_argmax(H): table = pd.DataFrame({'cluster': indices, 'super_topic':indices, 'counts': counts, 'percentage':np.round(100*arr, 2)}) return labels, counts, table -def term_frequency(df, term, col): - term = term.lower() - return df[col].str.lower().str.count(term).sum() - -def document_frequency(df, term, col): - term = term.lower() - return df[col].apply(lambda x: term in x.lower() if pd.notnull(x) else False).sum() - -def calculate_term_representations(df, terms, col): - """ - Calculate term frequency, document frequency, and TF-IDF scores - for a list of terms in a pandas DataFrame. - - Parameters: - ----------- - df: pd.DataFrame - Pandas DataFrame with a column col containing the text data. - col: str - The column in which to search for terms - terms: list - List of terms to calculate TF-IDF scores, term frequency, and document frequency for. - - Returns: - -------- - pd.DataFrame - A new DataFrame with columns 'Term', 'Term Frequency', 'Document Frequency', 'TF-IDF Score'. - """ - vectorizer = TfidfVectorizer(vocabulary=terms, dtype=np.float32) - tfidf_matrix = vectorizer.fit_transform(df[col].dropna()) - - # get feature names (the terms vocabulary) from vectorizer - feature_names = list(vectorizer.get_feature_names_out()) - - # calculate average TF-IDF score for each term - avg_tfidf_scores = tfidf_matrix.mean(axis=0).tolist()[0] - - # prepare results DataFrame - results_df = { - 'Term': [], - 'Term Frequency': [], - 'Document Frequency': [], - 'TF-IDF Score': [] - } - - # calculate TF, DF for each term - for term in terms: - term_index = feature_names.index(term) - tf = term_frequency(df, term, col) - df_count = document_frequency(df, term, col) - tfidf_score = avg_tfidf_scores[term_index] - - results_df['Term'].append(term) - results_df['Term Frequency'].append(tf) - results_df['Document Frequency'].append(df_count) - results_df['TF-IDF Score'].append(tfidf_score) - - return pd.DataFrame.from_dict(results_df) def best_n_papers(df, path, n): """ diff --git a/TELF/post_processing/Wolf/graph.py b/TELF/post_processing/Wolf/graph.py index fd581026..52f8de39 100644 --- a/TELF/post_processing/Wolf/graph.py +++ b/TELF/post_processing/Wolf/graph.py @@ -377,17 +377,41 @@ def get_hubs_authorities(self, max_iter=100): Parameters ---------- max_iter: integer; optional - Maximum number of iterations in power method. Default=None. + Maximum number of iterations in power method. Default=100. Returns ------- dict Dictionary with hubs, authorities, and the graph uid as values """ - hubs, authorities = nx.hits(self.G, max_iter=max_iter) + n_nodes = self.G.number_of_nodes() + + # Handle trivial graphs + if n_nodes == 0: + hubs = {} + authorities = {} + elif n_nodes == 1: + only_node = next(iter(self.G.nodes())) + hubs = {only_node: 1.0} + authorities = {only_node: 1.0} + else: + try: + hubs, authorities = nx.hits(self.G, max_iter=max_iter) + except ValueError: + # fallback if svds fails for unexpected reasons + hubs = {n: 0.0 for n in self.G.nodes()} + authorities = hubs.copy() + + # record in stats self.stats["hubs"] = hubs self.stats["authorities"] = authorities - return {"hubs": hubs, "authorities": authorities, "uid": self.uid} + + return { + "hubs": hubs, + "authorities": authorities, + "uid": self.uid + } + def strongly_connected_components(self): diff --git a/TELF/post_processing/Wolf/plots.py b/TELF/post_processing/Wolf/plots.py index dccdacc5..ca33f0ff 100644 --- a/TELF/post_processing/Wolf/plots.py +++ b/TELF/post_processing/Wolf/plots.py @@ -4,7 +4,14 @@ from tqdm import tqdm import plotly.graph_objs as go from .wolf import Wolf +from tqdm import tqdm +import os +import pandas as pd +from ...helpers.figures import create_wordcloud_from_df +from ...helpers.filters import find_subdf +from ...pre_processing.Vulture.tokens_analysis.top_words import get_top_words +from ...helpers.frames import calculate_term_representations def create_slice_graphs(X, node_ids=None, node_attributes=None, slice_ids=None, verbose=False): """ @@ -892,4 +899,92 @@ def plot_matrix_directed_graph(X, node_ids, *, node_attributes=None, filter_isol ) ) ) - return fig \ No newline at end of file + return fig + +def save_components(df, ranking_df, g, col, results_dir, terms=None, save_excel=False): + wcc = g.strongly_connected_components() + + component_table_df = { + 'component': [], + 'num_papers': [], + 'num_nodes': [], + } + + for i, c in tqdm(enumerate(wcc)): + output_dir = os.path.join(results_dir, str(i)) + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + # get author ids from graph + components_nodes = list(c.G.nodes) + + # get the graph rankings for the given component + component_node_df = ranking_df.loc[ranking_df.node_id.isin(set(components_nodes))] + component_node_df.to_csv(os.path.join(output_dir, f'component_{i}_with_{len(c)}_nodes.csv'), index=False) + if save_excel: + component_node_df.to_excel(os.path.join(output_dir, f'component_{i}_with_{len(c)}_nodes.xlsx'), index=False) + + # get the papers for the given component + component_document_df = find_subdf(df, col, components_nodes) + component_document_df.to_csv(os.path.join(output_dir, f'component_{i}_' \ + f'with_{len(component_document_df)}_documents.csv'), index=False) + + if save_excel: + component_document_df.to_excel(os.path.join(output_dir, f'component_{i}_' \ + f'with_{len(component_document_df)}_documents.xlsx'), index=False) + + # calculate attribution for component + if terms is not None: + attribution_df = calculate_term_representations(component_document_df, terms, col='cleaned_title_abstract') + attribution_df.to_csv(os.path.join(output_dir, f'component_{i}_attribution.csv'), index=False) + + if save_excel: + attribution_df.to_excel(os.path.join(output_dir, f'component_{i}_attribution.xlsx'), index=False) + + # build table + component_table_df['component'].append(i) + component_table_df['num_nodes'].append(len(c)) + component_table_df['num_papers'].append(len(component_document_df)) + + # visualize + num_nodes = len(component_node_df) + c.visualize(font_color = 'black', + node_color = '#edede9', + node_size = 200, + font_size = 8, + edge_width = 0.08, + highlight_nodes = None, + save_path = os.path.join(output_dir, f'component_{i}_with_{len(c)}_nodes.png'), + figsize = (6,6)) + + + component_table_df = pd.DataFrame.from_dict(component_table_df) + component_table_df.to_csv(os.path.join(results_dir, 'component_table.csv'), index=False) + + +def component_wordclouds(df, g, col, results_dir, top_words=30, text_col='clean_title_abstract'): + wcc = g.strongly_connected_components() + for i, c in tqdm(enumerate(wcc)): + output_dir = os.path.join(results_dir, str(i)) + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + # get author ids from graph + components_nodes = list(c.G.nodes) + + # get the papers for the given component + component_document_df = find_subdf(df, col, components_nodes) + + # create/save the wordcloud + create_wordcloud_from_df( + df = component_document_df, + col = text_col, + n = top_words, + save_path = os.path.join(output_dir, f'wordcloud_{i}.png'), + figsize = (6,6)) + + # create top n-grams files as well + documents = [x for x in component_document_df[text_col].to_list() if not pd.isna(x)] + for n in range(1,4): + ngrams = get_top_words(documents, top_n=100, n_gram=n, verbose=False) + ngrams.to_csv(os.path.join(output_dir, f'top_{n}-grams.csv'), index=False) \ No newline at end of file diff --git a/TELF/pre_processing/Orca/orca.py b/TELF/pre_processing/Orca/orca.py index 8a87b7a3..43bb3b53 100644 --- a/TELF/pre_processing/Orca/orca.py +++ b/TELF/pre_processing/Orca/orca.py @@ -658,8 +658,11 @@ def __compute_slic_scopus(self, df): affiliations = scopus_affiliations.get(eid) if affiliations is not None: for aff_id, aff_info_shallow in affiliations.items(): + if isinstance(aff_info_shallow, list): + continue del_list = [] # items to remove aff_info = copy.deepcopy(aff_info_shallow) + for i in range(len(aff_info['authors'])): scopus_id = str(aff_info['authors'][i]) if scopus_id not in scopus_to_slic: @@ -813,6 +816,8 @@ def __generate_affiliations_map(self, df): year = int(year) for aff_id, info in affiliations.items(): + if isinstance(info, list): + continue aff_name = info.get('name', 'Unknown') aff_country = info.get('country', 'Unknown') diff --git a/TELF/pre_processing/Orca/utils.py b/TELF/pre_processing/Orca/utils.py index 849c5960..67e7774e 100644 --- a/TELF/pre_processing/Orca/utils.py +++ b/TELF/pre_processing/Orca/utils.py @@ -1,7 +1,8 @@ import rapidfuzz import unicodedata from itertools import permutations - +import json +import pandas as pd def generate_name_variations(name): """ @@ -130,4 +131,55 @@ def match_name(name_a, name_b, normalize=False, scorer=rapidfuzz.distance.Indel. return 0 else: _, score, _ = fuzz_output - return score \ No newline at end of file + return score + + +def orca_summary(df, save_path=None): + rows = [] + + df_filtered = df.dropna(subset=['type']) + + for t in sorted(df_filtered['type'].unique()): + subset = df_filtered[df_filtered['type'] == t] + + # Unique slic_author_ids (split by ';') + unique_authors = set() + subset['slic_author_ids'].dropna().apply(lambda x: unique_authors.update(x.split(';'))) + + # Unique slic_affiliations (as dict keys) + unique_affils = set() + for aff in subset['slic_affiliations'].dropna(): + if isinstance(aff, str): + try: + aff = json.loads(aff.replace("'", '"')) # fix improperly formatted dicts + except: + continue + if isinstance(aff, dict): + unique_affils.update(aff.keys()) + + # Unique countries from affiliations + unique_countries = set() + for a in subset['affiliations'].dropna(): + try: + aff_dict = json.loads(a.replace("'", '"')) + for aff in aff_dict.values(): + if isinstance(aff, dict) and 'country' in aff: + unique_countries.add(aff['country']) + except Exception: + continue + + rows.append({ + 'type': t, + 'count': len(subset), + 'unique_slic_author_ids': len(unique_authors), + 'unique_slic_affiliations': len(unique_affils), + 'unique_countries': len(unique_countries) + }) + + summary_df = pd.DataFrame(rows) + + # Save to CSV + if save_path: + summary_df.to_csv(save_path , index=False) # "type_summary.csv" + + return summary_df diff --git a/TELF/pre_processing/Squirrel/pruners/llm_prune.py b/TELF/pre_processing/Squirrel/pruners/llm_prune.py index 37ad8c8b..407afd01 100644 --- a/TELF/pre_processing/Squirrel/pruners/llm_prune.py +++ b/TELF/pre_processing/Squirrel/pruners/llm_prune.py @@ -2,12 +2,13 @@ import logging import math from pathlib import Path -from typing import Union - +from typing import Callable, Iterable import pandas as pd from tqdm.auto import tqdm -from ....helpers.llm import get_ollama_llm, vote_once, build_json_vote_prompt +from ....helpers.llm_operations import vote_once +from ....helpers.llm_models import get_ollama_llm +from ....helpers.prompts import build_json_vote_prompt log = logging.getLogger(__name__) @@ -21,6 +22,7 @@ def __init__( llm_promote_threshold: float, llm_temperature: float, verbose: bool = True, + prompt: Callable[[str, Iterable[str]], str] = build_json_vote_prompt ): """ Perform LLM-based refinement on an embedding-pruned dataset, annotating each @@ -40,11 +42,17 @@ def __init__( Sampling temperature for the LLM. verbose : bool Whether to show tqdm progress bars. + prompt : Callable[[str, Iterable[str]], str] + Function that contains the prompt for the LLM.\n + This function should take in a string and iterable where the string is the candidate document\n + and the iterable is the context or target context examples to compare the candidate docuiment to.\n + Default is ``TELF.helpers.prompts.build_json_vote_prompt`` """ self.llm_vote_trials = llm_vote_trials self.llm_promote_threshold = llm_promote_threshold self.verbose = verbose self.NAME = 'llm_prune' + self.prompt = prompt self.llm = get_ollama_llm( model=llm_model_name, @@ -73,7 +81,7 @@ def _vote_on_document(self, df, idx, row, ctx, keep, fout, data_column) -> None: data_column : str Column name containing the data to be voted on. """ - prompt = build_json_vote_prompt(row[data_column], ctx) + prompt = self.prompt(row[data_column], ctx) votes = [vote_once(self.llm, prompt) for _ in range(self.llm_vote_trials)] yes_count = sum(int(v[0]) for v in votes) reasons = [v[1] for v in votes] diff --git a/TELF/pre_processing/Vulture/tokens_analysis/top_words.py b/TELF/pre_processing/Vulture/tokens_analysis/top_words.py index 334985c3..71896721 100644 --- a/TELF/pre_processing/Vulture/tokens_analysis/top_words.py +++ b/TELF/pre_processing/Vulture/tokens_analysis/top_words.py @@ -45,14 +45,15 @@ def get_top_words(documents, word_stats = defaultdict(lambda: {"tf": 0, "df": 0}) for doc in tqdm(documents, disable=not verbose): - tokens = doc.split() - ngrams = zip(*[tokens[i:] for i in range(n_gram)]) - ngrams = [" ".join(ngram) for ngram in ngrams] + if isinstance(doc, str): + tokens = doc.split() + ngrams = zip(*[tokens[i:] for i in range(n_gram)]) + ngrams = [" ".join(ngram) for ngram in ngrams] - for gram in ngrams: - word_stats[gram]["tf"] += 1 - for gram in set(ngrams): - word_stats[gram]["df"] += 1 + for gram in ngrams: + word_stats[gram]["tf"] += 1 + for gram in set(ngrams): + word_stats[gram]["df"] += 1 word_stats = dict(word_stats) top_words = dict(sorted(word_stats.items(), key=lambda x: x[1]["tf"], reverse=True)[:top_n]) diff --git a/TELF/pre_processing/Vulture/vulture.py b/TELF/pre_processing/Vulture/vulture.py index 928c69c3..71f4cac1 100755 --- a/TELF/pre_processing/Vulture/vulture.py +++ b/TELF/pre_processing/Vulture/vulture.py @@ -110,7 +110,7 @@ class Vulture: ] - def __init__(self, *, n_jobs = -1, n_nodes = 1, parallel_backend = "multiprocessing", cache = '/tmp', verbose = False): + def __init__(self, *, n_jobs = -1, n_nodes = 1, parallel_backend = "threading", cache = '/tmp', verbose = False): self.n_jobs = n_jobs self.n_nodes = n_nodes self.parallel_backend = parallel_backend diff --git a/TELF/pre_processing/iPenguin/SemanticScholar/s2.py b/TELF/pre_processing/iPenguin/SemanticScholar/s2.py index 51ec4a97..46410fba 100644 --- a/TELF/pre_processing/iPenguin/SemanticScholar/s2.py +++ b/TELF/pre_processing/iPenguin/SemanticScholar/s2.py @@ -61,7 +61,7 @@ def get_df_helper(files): data['year'].append(year) authors = None - if contents is not None and 'authors' in contents: + if contents is not None and 'authors' in contents and contents["authors"] is not None: authors = [str(x.get('name', None)) for x in contents['authors']] authors = [x for x in authors if x is not None] if not authors: @@ -71,7 +71,7 @@ def get_df_helper(files): data['s2_authors'].append(authors) author_ids = None - if contents is not None and 'authors' in contents: + if contents is not None and 'authors' in contents and contents["authors"] is not None: author_ids = [str(x.get('authorId', None)) for x in contents['authors']] author_ids = [x for x in author_ids if x is not None] if not author_ids: @@ -82,7 +82,7 @@ def get_df_helper(files): citations = None num_citations = 0 - if contents is not None and 'citations' in contents: + if contents is not None and 'citations' in contents and contents["citations"] is not None: citations = [str(x.get('paperId', None)) for x in contents['citations']] citations = [x for x in citations if x is not None] if not citations: @@ -95,7 +95,7 @@ def get_df_helper(files): references = None num_references = 0 - if contents is not None and 'references' in contents: + if contents is not None and 'references' in contents and contents["references"] is not None: references = [str(x.get('paperId', None)) for x in contents['references']] references = [x for x in references if x is not None] if not references: diff --git a/TELF/version.py b/TELF/version.py index 205c9068..92ea29de 100644 --- a/TELF/version.py +++ b/TELF/version.py @@ -1 +1 @@ -__version__ = "0.0.41" +__version__ = "0.0.42" diff --git a/data/sample_doi.csv b/data/sample_doi.csv index a5d65c23..b87ee0b3 100644 --- a/data/sample_doi.csv +++ b/data/sample_doi.csv @@ -1,4 +1,26 @@ doi 10.1109/ISDFS60797.2024.10527250 10.48550/arXiv.2309.01350 -10.48550/arXiv.2503.04680 \ No newline at end of file +10.48550/arXiv.2503.04680 +10.1007/s10915-025-02860-x +10.48550/arXiv.2502.20364 +10.2514/6.2025-0304 +10.1109/ICMLA61862.2024.00258 +10.1109/ICMLA61862.2024.00085 +10.1109/HPEC62836.2024.10938517 +10.1145/3685650.3685667 +10.1109/ISDFS60797.2024.10527237 +10.1109/MILCOM58377.2023.10356348 +10.1109/ICMLA58977.2023.00148 +10.1109/ISI58743.2023.10297217 +10.1109/ICMLA55696.2022.00107 +10.1109/HPEC55821.2022.9926288 +10.1145/3558100.3563844 +10.1145/3469096.3474927 +10.1109/ISI49825.2020.9280524 +10.1145/3395027.3419591 +10.1145/3624567 +10.1007/s11227-023-05587-4 +10.1145/3519602 +10.1145/3386363 +10.1007/978-3-031-66245-4 1 \ No newline at end of file diff --git a/data/sample_terms.md b/data/sample_terms.md new file mode 100644 index 00000000..e6112a1c --- /dev/null +++ b/data/sample_terms.md @@ -0,0 +1 @@ +## malware family classification \ No newline at end of file diff --git a/data/sample_terms2.md b/data/sample_terms2.md new file mode 100644 index 00000000..169cd09a --- /dev/null +++ b/data/sample_terms2.md @@ -0,0 +1,9 @@ +## malware family +positives: classification +negatives: labeling, early +## decision-making +positives: reward +## pre-train model +positives: supervised +## pre-train +positives: supervised \ No newline at end of file diff --git a/docs/ArcticFox.html b/docs/ArcticFox.html index e3a3f9f8..1e010498 100644 --- a/docs/ArcticFox.html +++ b/docs/ArcticFox.html @@ -8,7 +8,7 @@ - TELF.post_processing.ArcticFox: Report generation tool for text data from HNMFk using local LLMs — TELF 0.0.41 documentation + TELF.post_processing.ArcticFox: Report generation tool for text data from HNMFk using local LLMs — TELF 0.0.42 documentation @@ -30,7 +30,7 @@ - + @@ -40,7 +40,7 @@ - + @@ -126,7 +126,7 @@ -

TELF 0.0.41 documentation

+

TELF 0.0.42 documentation

diff --git a/docs/Beaver.html b/docs/Beaver.html index 8440ad0f..d1f900d2 100644 --- a/docs/Beaver.html +++ b/docs/Beaver.html @@ -8,7 +8,7 @@ - TELF.pre_processing.Beaver: Fast matrix and tensor building tool — TELF 0.0.41 documentation + TELF.pre_processing.Beaver: Fast matrix and tensor building tool — TELF 0.0.42 documentation @@ -30,7 +30,7 @@ - + @@ -40,7 +40,7 @@ - + @@ -126,7 +126,7 @@ -

TELF 0.0.41 documentation

+

TELF 0.0.42 documentation

@@ -409,14 +409,14 @@

TELF.pre_processing.Beaver: Fast matrix and tensor building

Example#

Several examples for Beaver can be found here. Here we will show example usage for creating a Document-Words matrix. First let’s load the example dataset (example dataset can be found here):

-
import pandas as pd
+
import pandas as pd
 
 df = pd.read_csv("../../data/sample.csv")
 df.info()
 

 

 
Next, let’s build the vocabulary:

-
from TELF.pre_processing import Beaver
+
from TELF.pre_processing import Beaver
 
 beaver = Beaver()
 settings = {
@@ -444,7 +444,7 @@ 
Example#<
 

 

 
We cam then load the matrix:

-
import scipy.sparse as ss
+
import scipy.sparse as ss
 
 # load the saved file which is in Sparse COO format
 X_csr_sparse = ss.load_npz("documents_words.npz")
diff --git a/docs/Bunny.html b/docs/Bunny.html
index 6ca65074..fa0f700f 100644
--- a/docs/Bunny.html
+++ b/docs/Bunny.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.applications.Bunny: Dataset generation tool for documents and their citations/references — TELF 0.0.41 documentation
+    TELF.applications.Bunny: Dataset generation tool for documents and their citations/references — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
TELF 0.0.41 documentation

+    
TELF 0.0.42 documentation

   
 

         

diff --git a/docs/Cheetah.html b/docs/Cheetah.html
index 8156cf45..8307993f 100644
--- a/docs/Cheetah.html
+++ b/docs/Cheetah.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.applications.Cheetah: Advanced search by keywords and phrases — TELF 0.0.41 documentation
+    TELF.applications.Cheetah: Advanced search by keywords and phrases — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
TELF 0.0.41 documentation

+    
TELF 0.0.42 documentation

   
 

         

diff --git a/docs/Fox.html b/docs/Fox.html
index 38ee59ea..64a633de 100644
--- a/docs/Fox.html
+++ b/docs/Fox.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.post_processing.Fox: Report generation tool for text data from HNMFk using local LLMs — TELF 0.0.41 documentation
+    TELF.post_processing.Fox: Report generation tool for text data from HNMFk using local LLMs — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
TELF 0.0.41 documentation

+    
TELF 0.0.42 documentation

   
 

         

diff --git a/docs/HNMFk.html b/docs/HNMFk.html
index 46d515dc..bd2c472f 100644
--- a/docs/HNMFk.html
+++ b/docs/HNMFk.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization.HNMFk: Hierarchical Non-negative Matrix Factorization with Automatic Model Determination — TELF 0.0.41 documentation
+    TELF.factorization.HNMFk: Hierarchical Non-negative Matrix Factorization with Automatic Model Determination — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
TELF 0.0.41 documentation

+    
TELF 0.0.42 documentation

   
 

         

diff --git a/docs/NMFk.html b/docs/NMFk.html
index 78a73a25..56cee12e 100644
--- a/docs/NMFk.html
+++ b/docs/NMFk.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization.NMFk: Non-negative Matrix Factorization with Automatic Model Determination — TELF 0.0.41 documentation
+    TELF.factorization.NMFk: Non-negative Matrix Factorization with Automatic Model Determination — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
TELF 0.0.41 documentation

+    
TELF 0.0.42 documentation

   
 

         

@@ -397,15 +397,15 @@ 
TELF.factorization.NMFk: Non-negative Matrix Factorization
 

 
Example#

 
First generate synthetic data with pre-determined k. It can be either dense (np.ndarray) or sparse matrix (scipy.sparse._csr.csr_matrix). Here, we are using the provided scripts for matrix generation (located here):

-
import sys; sys.path.append("../../scripts/")
-from generate_X import gen_data,gen_data_sparse
+
import sys; sys.path.append("../../scripts/")
+from generate_X import gen_data,gen_data_sparse
 
 Xsp = gen_data_sparse(shape=[100, 200], density=0.01)["X"]
 X = gen_data(R=4, shape=[100, 200])["X"]
 

 

 
Now we can factorize the given matrix:

-
from TELF.factorization import NMFk
+
from TELF.factorization import NMFk
 
 params = {
     "n_perturbs":36,
diff --git a/docs/Orca.html b/docs/Orca.html
index 1a738ce8..8ebbac15 100644
--- a/docs/Orca.html
+++ b/docs/Orca.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.pre_processing.Orca: Duplicate author detector for text mining and information retrival — TELF 0.0.41 documentation
+    TELF.pre_processing.Orca: Duplicate author detector for text mining and information retrival — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
TELF 0.0.41 documentation

+    
TELF 0.0.42 documentation

   
 

         

diff --git a/docs/Penguin.html b/docs/Penguin.html
index 17ecb031..0f4d1dc8 100644
--- a/docs/Penguin.html
+++ b/docs/Penguin.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.applications.Penguin: Text storage tool — TELF 0.0.41 documentation
+    TELF.applications.Penguin: Text storage tool — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
TELF 0.0.41 documentation

+    
TELF 0.0.42 documentation

   
 

         

@@ -381,6 +381,8 @@ 
 Contents 

 
  • Penguin.citation_search()
    • 
 
  • Penguin.count_documents()
    • 
 
  • Penguin.db_name
    • 
+
  • Penguin.distinct_dois()
    • 
+
  • Penguin.distinct_field()
    • 
 
  • Penguin.find_by_tag()
    • 
 
  • Penguin.get_id_bloom()
    • 
 
  • Penguin.id_search()
    • 
@@ -697,6 +699,19 @@ 
    Raises:#
 property db_name#
 
    
 
+
    
+
    
+distinct_dois(source: str = 'S2', dois: Sequence[str] | None = None) → set[source]#
    
+
    Return the subset of dois that are already in the database (or all DOIs if dois is None).
    
+
    
+
+
    
+
    
+distinct_field(source: str, field: str, filter_spec: dict | None = None) → set[source]#
    
+
    Return the set of unique values for field in source matching filter_spec.
+If filter_spec is None, returns all distinct values of that field.
    
+
    
+
 
    
 
    
 find_by_tag(tag, as_pandas=True)[source]#
    
@@ -1104,6 +1119,8 @@ 
    Example:#Penguin.citation_search()
 
  • Penguin.count_documents()
    • 
 
  • Penguin.db_name
    • 
+
  • Penguin.distinct_dois()
    • 
+
  • Penguin.distinct_field()
    • 
 
  • Penguin.find_by_tag()
    • 
 
  • Penguin.get_id_bloom()
    • 
 
  • Penguin.id_search()
    • 
diff --git a/docs/RESCALk.html b/docs/RESCALk.html
index dfaaf1bd..d54df646 100644
--- a/docs/RESCALk.html
+++ b/docs/RESCALk.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization.RESCALk: RESCAL with Automatic Model Determination — TELF 0.0.41 documentation
+    TELF.factorization.RESCALk: RESCAL with Automatic Model Determination — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
@@ -397,15 +397,15 @@ 
    TELF.factorization.RESCALk: RESCAL with Automatic Model Det
 
    
 
    Example#
    
 
    First generate synthetic data with pre-determined k. It can be either dense (np.ndarray) or sparse matrix (scipy.sparse._csr.csr_matrix). Here, we are using the provided scripts for matrix generation (located here):
    
-
    import sys;
+
    import sys;
 sys.path.append("../../scripts/")
-from generate_X import gen_data
+from generate_X import gen_data
 
 X = gen_data(R=4, shape=[1000,1000,8],gen='rescal')["X"]
 
    
 
    
 
    Now we can factorize the given matrix:
    
-
    from TELF.factorization import RESCALk
+
    from TELF.factorization import RESCALk
 
 params = {
     "n_perturbs": 2,
diff --git a/docs/SPLIT.html b/docs/SPLIT.html
index 4e1f7e3d..6278c304 100644
--- a/docs/SPLIT.html
+++ b/docs/SPLIT.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization.SPLIT: Joint NMFk factorization of multiple data via SPLIT — TELF 0.0.41 documentation
+    TELF.factorization.SPLIT: Joint NMFk factorization of multiple data via SPLIT — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/SPLITTransfer.html b/docs/SPLITTransfer.html
index c85dfbd1..7c89f81b 100644
--- a/docs/SPLITTransfer.html
+++ b/docs/SPLITTransfer.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization.SPLITTransfer: Supervised transfer learning method via SPLIT and NMFk — TELF 0.0.41 documentation
+    TELF.factorization.SPLITTransfer: Supervised transfer learning method via SPLIT and NMFk — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/SeaLion.html b/docs/SeaLion.html
index d0c51cd6..a035794d 100644
--- a/docs/SeaLion.html
+++ b/docs/SeaLion.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.post_processing.SeaLion: Generic report generation tool — TELF 0.0.41 documentation
+    TELF.post_processing.SeaLion: Generic report generation tool — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/Squirrel.html b/docs/Squirrel.html
index cf6cce71..c915c343 100644
--- a/docs/Squirrel.html
+++ b/docs/Squirrel.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.pre_processing.Squirrel: Dataset pruning tool — TELF 0.0.41 documentation
+    TELF.pre_processing.Squirrel: Dataset pruning tool — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
@@ -534,7 +534,7 @@ 
    Available Functions#
    
 
    
 
    
-class TELF.pre_processing.Squirrel.pruners.llm_prune.LLMPruner(llm_model_name: str, llm_api_url: str, llm_vote_trials: int, llm_promote_threshold: float, llm_temperature: float, verbose: bool = True)[source]#
    
+class TELF.pre_processing.Squirrel.pruners.llm_prune.LLMPruner(llm_model_name: str, llm_api_url: str, llm_vote_trials: int, llm_promote_threshold: float, llm_temperature: float, verbose: bool = True, prompt: ~typing.Callable[[str, ~typing.Iterable[str]], str] = <function build_json_vote_prompt>)[source]#
 
    Bases: object
    
 
    Perform LLM-based refinement on an embedding-pruned dataset, annotating each
 document with an llm_accept boolean column.
    
@@ -547,6 +547,11 @@ 
    Module ContentsTELF.helpers.prompts.build_json_vote_prompt
    
+
    
 
 
    
 
    
diff --git a/docs/SymNMFk.html b/docs/SymNMFk.html
index a62e1f22..43283753 100644
--- a/docs/SymNMFk.html
+++ b/docs/SymNMFk.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization.SymNMFk: Symmetric Non-negative Matrix Factorization with Automatic Model Determination — TELF 0.0.41 documentation
+    TELF.factorization.SymNMFk: Symmetric Non-negative Matrix Factorization with Automatic Model Determination — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/TELF.factorization.decompositions.html b/docs/TELF.factorization.decompositions.html
index 56231513..ad60a3f5 100644
--- a/docs/TELF.factorization.decompositions.html
+++ b/docs/TELF.factorization.decompositions.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization.decompositions package — TELF 0.0.41 documentation
+    TELF.factorization.decompositions package — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -127,7 +127,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/TELF.factorization.decompositions.utilities.html b/docs/TELF.factorization.decompositions.utilities.html
index f0624924..69ef9d75 100644
--- a/docs/TELF.factorization.decompositions.utilities.html
+++ b/docs/TELF.factorization.decompositions.utilities.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization.decompositions.utilities package — TELF 0.0.41 documentation
+    TELF.factorization.decompositions.utilities package — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/TELF.factorization.html b/docs/TELF.factorization.html
index 4d03c2c6..e500d58f 100644
--- a/docs/TELF.factorization.html
+++ b/docs/TELF.factorization.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization package — TELF 0.0.41 documentation
+    TELF.factorization package — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -127,7 +127,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/TELF.factorization.utilities.html b/docs/TELF.factorization.utilities.html
index 7f64848a..44f8b0de 100644
--- a/docs/TELF.factorization.utilities.html
+++ b/docs/TELF.factorization.utilities.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization.utilities package — TELF 0.0.41 documentation
+    TELF.factorization.utilities package — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/TELF.html b/docs/TELF.html
index 1ed8b482..ebb6798d 100644
--- a/docs/TELF.html
+++ b/docs/TELF.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF package — TELF 0.0.41 documentation
+    TELF package — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -127,7 +127,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/TELF.pre_processing.Beaver.html b/docs/TELF.pre_processing.Beaver.html
index 6d2f3d9e..fb870bd6 100644
--- a/docs/TELF.pre_processing.Beaver.html
+++ b/docs/TELF.pre_processing.Beaver.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.pre_processing.Beaver package — TELF 0.0.41 documentation
+    TELF.pre_processing.Beaver package — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/TELF.pre_processing.Vulture.html b/docs/TELF.pre_processing.Vulture.html
index 78bb304f..a3397953 100644
--- a/docs/TELF.pre_processing.Vulture.html
+++ b/docs/TELF.pre_processing.Vulture.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.pre_processing.Vulture package — TELF 0.0.41 documentation
+    TELF.pre_processing.Vulture package — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
@@ -440,7 +440,7 @@ 
    Submodules
 
    
-class TELF.pre_processing.Vulture.vulture.Vulture(*, n_jobs=-1, n_nodes=1, parallel_backend='multiprocessing', cache='/tmp', verbose=False)[source]#
    
+class TELF.pre_processing.Vulture.vulture.Vulture(*, n_jobs=-1, n_nodes=1, parallel_backend='threading', cache='/tmp', verbose=False)[source]#
 
    Bases: object
    
 
    Vulture is a parallel, multi-node parallel, and distributed parallel document
 pre-processing tool.
@@ -453,7 +453,7 @@ 
    Submodules
 
    
-DEFAULT_PIPELINE = [SimpleCleaner(module_type='CLEANER', effective_stop_words=['characteristics', 'acknowledgment', 'characteristic', 'corresponding', 'unfortunately', 'automatically', 'approximately', 'substantially', 'investigation', 'significantly', 'predominantly', 'applications', 'particularly', 'respectively', 'consequently', 'introduction', 'representing', 'demonstrated', 'demonstrates', 'successfully', 'nevertheless', 'sufficiently', 'specifically', 'conclusions', 'performance', ... (+1359 more)], patterns={'standardize_hyphens': (re.compile('[\\u002D\\u2010\\u2011\\u2012\\u2013\\u2014\\u2015\\u2212\\u2E3A\\u2E3B]'), '-'), 'remove_copyright_statement': None, 'remove_stop_phrases': None, 'make_lower_case': None, 'normalize': None, 'remove_trailing_dash': ('(?<!\\w)-|-(?!\\w)', ''), 'make_hyphens_words': ('([a-z])\\-([a-z])', ''), 'remove_next_line': ('\\n+', ' '), 'remove_email': ('\\S*@\\S*\\s?', ''), 'remove_formulas': ('\\b\\w*[\\=\\≈\\/\\\\\\±]\\w*\\b', ''), 'remove_dash': ('-', ''), 'remove_between_[]': ('\\[.*?\\]', ' '), 'remove_between_()': ('\\(.*?\\)', ' '), 'remove_[]': ('[\\[\\]]', ' '), 'remove_()': ('[()]', ' '), 'remove_\\': ('\\\\', ' '), 'remove_numbers': ('\\b\\d+\\b', ''), 'remove_standalone_numbers': ('\\b\\d+\\b', ''), 'remove_nonASCII_boundary': ('\\b[^\\x00-\\x7F]+\\b', ''), 'remove_nonASCII': ('[^\\x00-\\x7F]+', ''), 'remove_tags': ('&lt;/?.*?&gt;', ''), 'remove_special_characters': ('[!|"|#|$|%|&|\\|\\\'|(|)|*|+|,|.|/|:|;|<|=|>|?|@|[|\\|]|^|_|`|{|\\||}|~]', ''), 'isolate_frozen': None, 'remove_extra_whitespace': ('\\s+', ' '), 'remove_stop_words': None, 'min_characters': None, 'remove_alphanumeric': ('\\b(?=\\w*[A-Za-z])(?=\\w*\\d)\\w+\\b', ''), 'remove_roman_numerals': ('(?i)\\bM{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})\\b', '')}, exclude_hyphenated_stopwords=False, sw_pattern=re.compile('\\b[\\w-]+\\b'))]#
    
+DEFAULT_PIPELINE = [SimpleCleaner(module_type='CLEANER', effective_stop_words=['characteristics', 'characteristic', 'acknowledgment', 'unfortunately', 'predominantly', 'significantly', 'investigation', 'automatically', 'corresponding', 'approximately', 'substantially', 'demonstrated', 'applications', 'demonstrates', 'successfully', 'specifically', 'nevertheless', 'respectively', 'particularly', 'consequently', 'sufficiently', 'introduction', 'representing', 'researchers', 'potentially', ... (+1359 more)], patterns={'standardize_hyphens': (re.compile('[\\u002D\\u2010\\u2011\\u2012\\u2013\\u2014\\u2015\\u2212\\u2E3A\\u2E3B]'), '-'), 'remove_copyright_statement': None, 'remove_stop_phrases': None, 'make_lower_case': None, 'normalize': None, 'remove_trailing_dash': ('(?<!\\w)-|-(?!\\w)', ''), 'make_hyphens_words': ('([a-z])\\-([a-z])', ''), 'remove_next_line': ('\\n+', ' '), 'remove_email': ('\\S*@\\S*\\s?', ''), 'remove_formulas': ('\\b\\w*[\\=\\≈\\/\\\\\\±]\\w*\\b', ''), 'remove_dash': ('-', ''), 'remove_between_[]': ('\\[.*?\\]', ' '), 'remove_between_()': ('\\(.*?\\)', ' '), 'remove_[]': ('[\\[\\]]', ' '), 'remove_()': ('[()]', ' '), 'remove_\\': ('\\\\', ' '), 'remove_numbers': ('\\b\\d+\\b', ''), 'remove_standalone_numbers': ('\\b\\d+\\b', ''), 'remove_nonASCII_boundary': ('\\b[^\\x00-\\x7F]+\\b', ''), 'remove_nonASCII': ('[^\\x00-\\x7F]+', ''), 'remove_tags': ('&lt;/?.*?&gt;', ''), 'remove_special_characters': ('[!|"|#|$|%|&|\\|\\\'|(|)|*|+|,|.|/|:|;|<|=|>|?|@|[|\\|]|^|_|`|{|\\||}|~]', ''), 'isolate_frozen': None, 'remove_extra_whitespace': ('\\s+', ' '), 'remove_stop_words': None, 'min_characters': None, 'remove_alphanumeric': ('\\b(?=\\w*[A-Za-z])(?=\\w*\\d)\\w+\\b', ''), 'remove_roman_numerals': ('(?i)\\bM{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})\\b', '')}, exclude_hyphenated_stopwords=False, sw_pattern=re.compile('\\b[\\w-]+\\b'))]#
 
    
 
 
    
diff --git a/docs/TELF.pre_processing.Vulture.tokens_analysis.html b/docs/TELF.pre_processing.Vulture.tokens_analysis.html
index 0e36fca5..97a03732 100644
--- a/docs/TELF.pre_processing.Vulture.tokens_analysis.html
+++ b/docs/TELF.pre_processing.Vulture.tokens_analysis.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.pre_processing.Vulture.tokens_analysis package — TELF 0.0.41 documentation
+    TELF.pre_processing.Vulture.tokens_analysis package — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -125,7 +125,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/TELF.pre_processing.html b/docs/TELF.pre_processing.html
index 623cd3d4..4fc067eb 100644
--- a/docs/TELF.pre_processing.html
+++ b/docs/TELF.pre_processing.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.pre_processing package — TELF 0.0.41 documentation
+    TELF.pre_processing package — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/TriNMFk.html b/docs/TriNMFk.html
index 323078d6..e9174ec1 100644
--- a/docs/TriNMFk.html
+++ b/docs/TriNMFk.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.factorization.TriNMFk: NMFk with Automatic Determination of Latent Clusters and Patterns — TELF 0.0.41 documentation
+    TELF.factorization.TriNMFk: NMFk with Automatic Determination of Latent Clusters and Patterns — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
@@ -398,9 +398,9 @@ 
    TELF.factorization.TriNMFk: NMFk with Automatic Determinati
 
    
 
    Example#
    
 
    First generate synthetic data with pre-determined k. It can be either dense (np.ndarray) or sparse matrix (scipy.sparse._csr.csr_matrix). Here, we are using the provided scripts for matrix generation (located here):
    
-
    import sys; sys.path.append("../../scripts/")
-from generate_X import gen_trinmf_data
-import matplotlib.pyplot as plt
+
    import sys; sys.path.append("../../scripts/")
+from generate_X import gen_trinmf_data
+import matplotlib.pyplot as plt
 
 kwkh=(5,3)
 shape=(20,20)
@@ -458,7 +458,7 @@ 
    Example#<
     "verbose":True
 }
 
-from TELF.factorization import TriNMFk
+from TELF.factorization import TriNMFk
 model = TriNMFk(**tri_nmfk_params)
 
    
 
    
diff --git a/docs/Vulture.html b/docs/Vulture.html
index dc21c0e1..e5dbed7a 100644
--- a/docs/Vulture.html
+++ b/docs/Vulture.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.pre_processing.Vulture: Advanced text pre-processing and cleaning tool for NLP and text-mining — TELF 0.0.41 documentation
+    TELF.pre_processing.Vulture: Advanced text pre-processing and cleaning tool for NLP and text-mining — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
@@ -410,7 +410,7 @@ 
    TELF.pre_processing.Vulture: Advanced text pre-processing a
 
    
 
    Example#
    
 
    First let’s load the example dataset (example dataset can be found here):
    
-
    import pickle
+
    import pickle
 
 DATA_DIR = os.path.join('..', '..', 'data')
 DATA_DIR = pathlib.Path(DATA_DIR).resolve()
@@ -420,19 +420,19 @@ 
    Example#<
 
    
 
    Now we can perform pre-processing:
    
 
    # import libraries
-import os
-import pathlib
-import pandas as pd
+import os
+import pathlib
+import pandas as pd
 
-from TELF.pre_processing import Vulture
+from TELF.pre_processing import Vulture
 
-from TELF.pre_processing.Vulture.modules import SimpleCleaner
-from TELF.pre_processing.Vulture.modules import LemmatizeCleaner
-from TELF.pre_processing.Vulture.modules import SubstitutionCleaner
-from TELF.pre_processing.Vulture.modules import RemoveNonEnglishCleaner
+from TELF.pre_processing.Vulture.modules import SimpleCleaner
+from TELF.pre_processing.Vulture.modules import LemmatizeCleaner
+from TELF.pre_processing.Vulture.modules import SubstitutionCleaner
+from TELF.pre_processing.Vulture.modules import RemoveNonEnglishCleaner
 
-from TELF.pre_processing.Vulture.default_stop_words import STOP_WORDS
-from TELF.pre_processing.Vulture.default_stop_phrases import STOP_PHRASES
+from TELF.pre_processing.Vulture.default_stop_words import STOP_WORDS
+from TELF.pre_processing.Vulture.default_stop_phrases import STOP_PHRASES
 
 
 # output directory
@@ -513,7 +513,7 @@ 
    Available Functions
 
    
-class TELF.pre_processing.Vulture.vulture.Vulture(*, n_jobs=-1, n_nodes=1, parallel_backend='multiprocessing', cache='/tmp', verbose=False)[source]#
    
+class TELF.pre_processing.Vulture.vulture.Vulture(*, n_jobs=-1, n_nodes=1, parallel_backend='threading', cache='/tmp', verbose=False)[source]#
 
    Bases: object
    
 
    Vulture is a parallel, multi-node parallel, and distributed parallel document
 pre-processing tool.
@@ -526,7 +526,7 @@ 
    Available Functions
 
    
-DEFAULT_PIPELINE = [SimpleCleaner(module_type='CLEANER', effective_stop_words=['characteristics', 'acknowledgment', 'characteristic', 'corresponding', 'unfortunately', 'automatically', 'approximately', 'substantially', 'investigation', 'significantly', 'predominantly', 'applications', 'particularly', 'respectively', 'consequently', 'introduction', 'representing', 'demonstrated', 'demonstrates', 'successfully', 'nevertheless', 'sufficiently', 'specifically', 'conclusions', 'performance', ... (+1359 more)], patterns={'standardize_hyphens': (re.compile('[\\u002D\\u2010\\u2011\\u2012\\u2013\\u2014\\u2015\\u2212\\u2E3A\\u2E3B]'), '-'), 'remove_copyright_statement': None, 'remove_stop_phrases': None, 'make_lower_case': None, 'normalize': None, 'remove_trailing_dash': ('(?<!\\w)-|-(?!\\w)', ''), 'make_hyphens_words': ('([a-z])\\-([a-z])', ''), 'remove_next_line': ('\\n+', ' '), 'remove_email': ('\\S*@\\S*\\s?', ''), 'remove_formulas': ('\\b\\w*[\\=\\≈\\/\\\\\\±]\\w*\\b', ''), 'remove_dash': ('-', ''), 'remove_between_[]': ('\\[.*?\\]', ' '), 'remove_between_()': ('\\(.*?\\)', ' '), 'remove_[]': ('[\\[\\]]', ' '), 'remove_()': ('[()]', ' '), 'remove_\\': ('\\\\', ' '), 'remove_numbers': ('\\b\\d+\\b', ''), 'remove_standalone_numbers': ('\\b\\d+\\b', ''), 'remove_nonASCII_boundary': ('\\b[^\\x00-\\x7F]+\\b', ''), 'remove_nonASCII': ('[^\\x00-\\x7F]+', ''), 'remove_tags': ('&lt;/?.*?&gt;', ''), 'remove_special_characters': ('[!|"|#|$|%|&|\\|\\\'|(|)|*|+|,|.|/|:|;|<|=|>|?|@|[|\\|]|^|_|`|{|\\||}|~]', ''), 'isolate_frozen': None, 'remove_extra_whitespace': ('\\s+', ' '), 'remove_stop_words': None, 'min_characters': None, 'remove_alphanumeric': ('\\b(?=\\w*[A-Za-z])(?=\\w*\\d)\\w+\\b', ''), 'remove_roman_numerals': ('(?i)\\bM{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})\\b', '')}, exclude_hyphenated_stopwords=False, sw_pattern=re.compile('\\b[\\w-]+\\b'))]#
    
+DEFAULT_PIPELINE = [SimpleCleaner(module_type='CLEANER', effective_stop_words=['characteristics', 'characteristic', 'acknowledgment', 'unfortunately', 'predominantly', 'significantly', 'investigation', 'automatically', 'corresponding', 'approximately', 'substantially', 'demonstrated', 'applications', 'demonstrates', 'successfully', 'specifically', 'nevertheless', 'respectively', 'particularly', 'consequently', 'sufficiently', 'introduction', 'representing', 'researchers', 'potentially', ... (+1359 more)], patterns={'standardize_hyphens': (re.compile('[\\u002D\\u2010\\u2011\\u2012\\u2013\\u2014\\u2015\\u2212\\u2E3A\\u2E3B]'), '-'), 'remove_copyright_statement': None, 'remove_stop_phrases': None, 'make_lower_case': None, 'normalize': None, 'remove_trailing_dash': ('(?<!\\w)-|-(?!\\w)', ''), 'make_hyphens_words': ('([a-z])\\-([a-z])', ''), 'remove_next_line': ('\\n+', ' '), 'remove_email': ('\\S*@\\S*\\s?', ''), 'remove_formulas': ('\\b\\w*[\\=\\≈\\/\\\\\\±]\\w*\\b', ''), 'remove_dash': ('-', ''), 'remove_between_[]': ('\\[.*?\\]', ' '), 'remove_between_()': ('\\(.*?\\)', ' '), 'remove_[]': ('[\\[\\]]', ' '), 'remove_()': ('[()]', ' '), 'remove_\\': ('\\\\', ' '), 'remove_numbers': ('\\b\\d+\\b', ''), 'remove_standalone_numbers': ('\\b\\d+\\b', ''), 'remove_nonASCII_boundary': ('\\b[^\\x00-\\x7F]+\\b', ''), 'remove_nonASCII': ('[^\\x00-\\x7F]+', ''), 'remove_tags': ('&lt;/?.*?&gt;', ''), 'remove_special_characters': ('[!|"|#|$|%|&|\\|\\\'|(|)|*|+|,|.|/|:|;|<|=|>|?|@|[|\\|]|^|_|`|{|\\||}|~]', ''), 'isolate_frozen': None, 'remove_extra_whitespace': ('\\s+', ' '), 'remove_stop_words': None, 'min_characters': None, 'remove_alphanumeric': ('\\b(?=\\w*[A-Za-z])(?=\\w*\\d)\\w+\\b', ''), 'remove_roman_numerals': ('(?i)\\bM{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})\\b', '')}, exclude_hyphenated_stopwords=False, sw_pattern=re.compile('\\b[\\w-]+\\b'))]#
 
    
 
 
    
diff --git a/docs/Wolf.html b/docs/Wolf.html
index 1079492b..3d01c3af 100644
--- a/docs/Wolf.html
+++ b/docs/Wolf.html
@@ -8,7 +8,7 @@
     
     
 
-    TELF.post_processing.Wolf: Graph centrality and ranking tool — TELF 0.0.41 documentation
+    TELF.post_processing.Wolf: Graph centrality and ranking tool — TELF 0.0.42 documentation
   
   
   
@@ -30,7 +30,7 @@
   
 
 
-    
+    
     
     
   
@@ -40,7 +40,7 @@
   
 
 
-    
+    
     
     
     
@@ -126,7 +126,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
diff --git a/docs/_modules/TELF/applications/Bunny/auto_bunny.html b/docs/_modules/TELF/applications/Bunny/auto_bunny.html
index f87c8442..2ee60823 100644
--- a/docs/_modules/TELF/applications/Bunny/auto_bunny.html
+++ b/docs/_modules/TELF/applications/Bunny/auto_bunny.html
@@ -7,7 +7,7 @@
   
     
     
-    TELF.applications.Bunny.auto_bunny — TELF 0.0.41 documentation
+    TELF.applications.Bunny.auto_bunny — TELF 0.0.42 documentation
   
   
   
@@ -29,7 +29,7 @@
   
 
 
-    
+    
     
     
   
@@ -39,7 +39,7 @@
   
 
 
-    
+    
     
     
     
@@ -125,7 +125,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
@@ -322,24 +322,24 @@ 
    
                 
    
                   
   
    Source code for TELF.applications.Bunny.auto_bunny
    -import os
-import re
-import sys
-import pathlib
-import pandas as pd
-from dataclasses import dataclass, field
-
-from .bunny import Bunny, BunnyFilter
-from ..Cheetah import Cheetah
-from ...pre_processing.iPenguin.Scopus import Scopus
-from ...pre_processing.iPenguin.SemanticScholar import SemanticScholar
-from ...pre_processing.Vulture import Vulture
-from ...helpers.file_system import check_path_var as check_path
+import os
+import re
+import sys
+import pathlib
+import pandas as pd
+from dataclasses import dataclass, field
+
+from .bunny import Bunny, BunnyFilter
+from ..Cheetah import Cheetah
+from ...pre_processing.iPenguin.Scopus import Scopus
+from ...pre_processing.iPenguin.SemanticScholar import SemanticScholar
+from ...pre_processing.Vulture import Vulture
+from ...helpers.file_system import check_path_var as check_path
 
 
    
 [docs]
 @dataclass
-class AutoBunnyStep:
+class AutoBunnyStep:
     """Class for keeping track of AutoBunny args"""
     modes: list
     max_papers: int = 0
@@ -351,7 +351,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    
 [docs]
-class AutoBunny:
+class AutoBunny:
     
     CHEETAH_INDEX = {
         'title': None, 
@@ -362,7 +362,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    'country': 'affiliations',
     }
     
-    def __init__(self, core, s2_key=None, scopus_keys=None, output_dir=None, cache_dir=None, cheetah_index=None, verbose=False):
+    def __init__(self, core, s2_key=None, scopus_keys=None, output_dir=None, cache_dir=None, cheetah_index=None, verbose=False):
         self.core = core
         self.s2_key = s2_key
         self.scopus_keys = scopus_keys
@@ -374,7 +374,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    
 [docs]
-    def run(self, 
+    def run(self, 
             steps, 
             *, 
             s2_key=None, 
@@ -459,7 +459,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    ### Helpers
     
     
-    def __final_cheetah_table(self, stem='cheetah_table'):
+    def __final_cheetah_table(self, stem='cheetah_table'):
         files = [x for x in os.listdir(self.output_dir) if x.endswith('.csv') and stem in x]
         frames = {}
         for f in files:
@@ -481,7 +481,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    return df
     
     
-    def __bunny_hop(self, df, modes, max_papers, hop_priority):
+    def __bunny_hop(self, df, modes, max_papers, hop_priority):
         bunny = Bunny(s2_key=self.s2_key, output_dir=self.cache_dir, verbose=self.verbose)
         use_scopus = self.scopus_keys is not None
         hop_df = bunny.hop(df, 1, modes, use_scopus=use_scopus, filters=None, max_papers=max_papers, hop_priority=hop_priority,
@@ -489,7 +489,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    return hop_df
     
     
-    def __cheetah_filter(self, df, cheetah_settings):
+    def __cheetah_filter(self, df, cheetah_settings):
     
         # index settings 
         cheetah_columns = {
@@ -524,12 +524,12 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    cheetah_table['included_ids'] = cheetah_table.included_ids.fillna('').str.split(';')\
                 .apply(lambda x: [int(i) for i in x if i] if x else [])
 
-            def include_s2ids(indices):
+            def include_s2ids(indices):
                 if not indices:
                     return None
                 return ';'.join(map(str, df_curr.loc[indices].s2id.to_list()))
             
-            def exclude_s2ids(indices):
+            def exclude_s2ids(indices):
                 all_s2ids = {x for x in df_curr.s2id.to_list() if not pd.isna(x)}
                 if not indices:
                     return ';'.join(list(all_s2ids))
@@ -547,7 +547,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    return cheetah_df, cheetah_table
     
     
-    def __vulture_clean(self, df, vulture_settings):
+    def __vulture_clean(self, df, vulture_settings):
         
         # setup vulture
         vulture = Vulture(n_jobs=-1, cache=self.output_dir, verbose=self.verbose)
@@ -568,31 +568,31 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    @property
-    def core(self):
+    def core(self):
         return self._core
     
     @property
-    def s2_key(self):
+    def s2_key(self):
         return self._s2_key
 
     @property
-    def scopus_keys(self):
+    def scopus_keys(self):
         return self._scopus_keys
 
     @property
-    def cheetah_index(self):
+    def cheetah_index(self):
         return self._cheetah_index
     
     @property
-    def output_dir(self):
+    def output_dir(self):
         return self._output_dir
     
     @property
-    def cache_dir(self):
+    def cache_dir(self):
         return self._cache_dir
     
     @core.setter
-    def core(self, core):
+    def core(self, core):
         if not isinstance(core, pd.DataFrame):
             raise ValueError('AutoBunny expects core to be a SLIC DataFrame!')
         if 'type' not in core:
@@ -600,7 +600,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    self._core = core
     
     @s2_key.setter
-    def s2_key(self, key):
+    def s2_key(self, key):
         if key is not None:
             self._s2_key = key
         elif isinstance(key, str):
@@ -613,7 +613,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    raise TypeError(f'Unsupported type "{type(key)}" for Semantic Scholar key')
         
     @scopus_keys.setter
-    def scopus_keys(self, scopus_keys):
+    def scopus_keys(self, scopus_keys):
         if scopus_keys is None:
             self._scopus_keys = scopus_keys
         elif isinstance(scopus_keys, (list, set)):
@@ -627,7 +627,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    raise TypeError(f'Unsupported type "{type(key)}" for Scopus key')
             
     @cheetah_index.setter
-    def cheetah_index(self, cheetah_index):
+    def cheetah_index(self, cheetah_index):
         if cheetah_index is None:
             self._cheetah_index = self.CHEETAH_INDEX
         elif isinstance(cheetah_index, dict):
@@ -640,7 +640,7 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    else:
             raise TypeError(f'Unsupported type "{type(cheetah_index)}" for `cheetah_index`')
 
-    def __process_path(self, path, var_name):
+    def __process_path(self, path, var_name):
         if path is None:
             return pathlib.Path('/tmp')
         elif isinstance(path, str):
@@ -653,11 +653,11 @@ 
    Source code for TELF.applications.Bunny.auto_bunny
    return _path
             
     @output_dir.setter
-    def output_dir(self, output_dir):
+    def output_dir(self, output_dir):
         self._output_dir = self.__process_path(output_dir, 'output_dir')
         
     @cache_dir.setter
-    def cache_dir(self, cache_dir):
+    def cache_dir(self, cache_dir):
         self._cache_dir = self.__process_path(cache_dir, 'cache_dir')
    
 
         
diff --git a/docs/_modules/TELF/applications/Bunny/bunny.html b/docs/_modules/TELF/applications/Bunny/bunny.html
index 377b35e1..18e3011b 100644
--- a/docs/_modules/TELF/applications/Bunny/bunny.html
+++ b/docs/_modules/TELF/applications/Bunny/bunny.html
@@ -7,7 +7,7 @@
   
     
     
-    TELF.applications.Bunny.bunny — TELF 0.0.41 documentation
+    TELF.applications.Bunny.bunny — TELF 0.0.42 documentation
   
   
   
@@ -29,7 +29,7 @@
   
 
 
-    
+    
     
     
   
@@ -39,7 +39,7 @@
   
 
 
-    
+    
     
     
     
@@ -125,7 +125,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
@@ -322,33 +322,33 @@ 
    
                 
    
                   
   
    Source code for TELF.applications.Bunny.bunny
    -import os
-import re
-import ast
-import sys
-import random
-import warnings
-import pandas as pd
-from typing import Union, List
-from collections import Counter
-from dataclasses import dataclass
+import os
+import re
+import ast
+import sys
+import random
+import warnings
+import pandas as pd
+from typing import Union, List
+from collections import Counter
+from dataclasses import dataclass
 pd.set_option('future.no_silent_downcasting', True)
 
-from ..Penguin import Penguin
-from ..Penguin.crocodile import form_df
-from ...pre_processing.iPenguin.Scopus import Scopus
-from ...pre_processing.iPenguin.SemanticScholar import SemanticScholar
-from ...pre_processing.iPenguin.utils import format_pubyear
-from ...pre_processing.Orca.AuthorMatcher import AuthorMatcher
+from ..Penguin import Penguin
+from ..Penguin.crocodile import form_df
+from ...pre_processing.iPenguin.Scopus import Scopus
+from ...pre_processing.iPenguin.SemanticScholar import SemanticScholar
+from ...pre_processing.iPenguin.utils import format_pubyear
+from ...pre_processing.Orca.AuthorMatcher import AuthorMatcher
 
 
    
 [docs]
 @dataclass
-class BunnyFilter:
+class BunnyFilter:
     filter_type: str
     filter_value: str
 
-    def __post_init__(self):
+    def __post_init__(self):
         if self.filter_type not in Bunny.FILTERS and self.filter_type != 'DOI':
             raise ValueError(f'Invalid filter_type "{self.filter_type}". Allowed values are {list(Bunny.FILTERS.keys())}')
    
 
@@ -357,11 +357,11 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
 @dataclass
-class BunnyOperation:
+class BunnyOperation:
     operator: str
     operands: List[Union[BunnyFilter, 'BunnyOperation']]
 
-    def __post_init__(self):
+    def __post_init__(self):
         if self.operator not in {"AND", "OR"}:
             raise ValueError(f'Invalid operator "{self.operator}". Only "AND" and "OR" are allowed.')
 
@@ -373,7 +373,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-def is_valid_query(query):
+def is_valid_query(query):
     """
     Validates the structure of a Bunny filter query.
 
@@ -408,7 +408,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-def evaluate_query_to_string(query):
+def evaluate_query_to_string(query):
     """
     Convert a Bunny query into a string that can be accepted by Scopus.
     
@@ -446,7 +446,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-def find_doi(f):
+def find_doi(f):
     """
     Helper function that attempts to extract the DOI from a Scopus paper XML file
     
@@ -477,7 +477,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-class Bunny():
+class Bunny():
     
     MODES = {'references', 'citations', 's2_author_ids'}
     FILTERS = {'AFFILCOUNTRY': 'Country', 
@@ -487,7 +487,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    'AU-ID': 'Scopus Author ID',
                'KEY': 'Keyword',}
     
-    def __init__(self, s2_key=None, scopus_keys=None, penguin_settings=None, output_dir='.', verbose=False):
+    def __init__(self, s2_key=None, scopus_keys=None, penguin_settings=None, output_dir='.', verbose=False):
         self.output_dir = output_dir
         self.verbose = verbose
         self.s2_key = s2_key
@@ -507,7 +507,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    }
 
     
-    def __init_lookup(self, series, priority, sep):
+    def __init_lookup(self, series, priority, sep):
         lookup = [y for x in series for y in x.split(sep)]
         lookup_freq = Counter(lookup)
         unique_lookup = list(lookup_freq.keys())
@@ -522,7 +522,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    return unique_lookup
         
     
-    def __hop(self, df, col, hop, hop_focus, s2_dir, sep, max_papers, hop_priority):
+    def __hop(self, df, col, hop, hop_focus, s2_dir, sep, max_papers, hop_priority):
         # setup access to Penguin cache
         ignore = None
         if self.penguin_settings is not None:
@@ -563,7 +563,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
     @classmethod
-    def estimate_hop(cls, df, col='citations'):
+    def estimate_hop(cls, df, col='citations'):
         """
         Predict the maximum number of papers that will be in the DataFrame if another hop is performed. 
         The numbers of papers in the next hop is guaranteed to be less than or equal to this number
@@ -587,7 +587,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
     @classmethod
-    def suggest_filter_values(cls, df, options=['AFFILCOUNTRY', 'AFFILORG']):
+    def suggest_filter_values(cls, df, options=['AFFILCOUNTRY', 'AFFILORG']):
         """
         Generate the possible Bunny filtering values for some options.
         Note that these values do not conclusively map all possible filter values used by Scopus
@@ -634,7 +634,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-    def form_core_scopus(self, data, data_type, keys, s2_dir='s2', scopus_dir='scopus'):
+    def form_core_scopus(self, data, data_type, keys, s2_dir='s2', scopus_dir='scopus'):
         if data_type not in ('scopus_author_id'):
             raise ValueError(f'Invalid data type: "{data_type}"')
         
@@ -683,7 +683,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-    def form_core(self, data, data_type, s2_dir='s2'):
+    def form_core(self, data, data_type, s2_dir='s2'):
         if not self.enabled:
             raise RuntimeError('Bunny object was not initialized with an S2 API key.')
         #if data_type not in ('doi', 's2id', 'bibtex', 's2_author_id'):
@@ -709,7 +709,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-    def hop(self, df, hops, modes, use_scopus=False, filters=None, hop_focus=None, scopus_keys=None, 
+    def hop(self, df, hops, modes, use_scopus=False, filters=None, hop_focus=None, scopus_keys=None, 
             s2_dir='s2', scopus_dir='scopus', filter_in_core=True, max_papers=0, hop_priority='random'):
         """
         Perform one or more hops along the citation network
@@ -806,7 +806,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    def __downloaded_scopus(self, dois, name, keys, filters):
+    def __downloaded_scopus(self, dois, name, keys, filters):
         ignore = None
         if self.penguin_settings is not None:
             ignore = self.get_penguin_cache(source='scopus')
@@ -837,7 +837,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    return df
     
 
-    def __verify_s2_df(self, df):
+    def __verify_s2_df(self, df):
         """
         Determine if the provided DataFrame matches the structure expected of a Bunny Semantic Scholar
         DataFrame.
@@ -855,7 +855,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    return True
 
 
-    def __verify_scopus_df(self, df):
+    def __verify_scopus_df(self, df):
         """
         Determine if the provided DataFrame matches the structure expected of a Bunny Scopus
         DataFrame.
@@ -875,7 +875,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-    def get_affiliations(self, df, scopus_keys, filters, save_dir='scopus', filter_in_core=True, do_author_match=True):
+    def get_affiliations(self, df, scopus_keys, filters, save_dir='scopus', filter_in_core=True, do_author_match=True):
         df_order = [
             'eid', 
             's2id', 
@@ -941,7 +941,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    def _filter_pubyear(self, df, f, auth_map=None):
+    def _filter_pubyear(self, df, f, auth_map=None):
         if 'year' not in df:
             raise ValueError('"year" not found in df')
 
@@ -960,7 +960,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    return pids
     
 
-    def _filter_key(self, df, f, auth_map=None):
+    def _filter_key(self, df, f, auth_map=None):
         keyword = f
         cols = ['title', 'abstract', 'PACs']
         mask = pd.Series(False, index=df.index)
@@ -972,13 +972,13 @@ 
    Source code for TELF.applications.Bunny.bunny
    return pids
 
 
-    def _filter_auid(self, df, f, auth_map=None):
+    def _filter_auid(self, df, f, auth_map=None):
         mask = pd.Series(False, index=df.index)
         pids = set(mask.index)
         return pids
 
     
-    def _filter_affil_generic(self, df, column_name, filter_value, auth_map): 
+    def _filter_affil_generic(self, df, column_name, filter_value, auth_map): 
         if 'affiliations' not in df:
             raise ValueError('"affiliations" not found in df')
 
@@ -1014,7 +1014,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    def _filter_affilorg(self, df, f, auth_map):
+    def _filter_affilorg(self, df, f, auth_map):
         if 'affiliations' not in df:
             raise ValueError('"affiliations" not found in df')
 
@@ -1041,7 +1041,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    return pids
         
 
-    def _filter_afid(self, df, f, auth_map):
+    def _filter_afid(self, df, f, auth_map):
         if 'affiliations' not in df:
             raise ValueError('"affiliations" not found in df')
 
@@ -1068,7 +1068,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    return pids
     
 
-    def __filter(self, df, filters, filter_in_core, do_author_match):
+    def __filter(self, df, filters, filter_in_core, do_author_match):
         all_pids = []
         if do_author_match and {'s2id', 'eid', 's2_author_ids', 's2_authors', 'authors', 'author_ids'}.issubset(df.columns):
             if self.verbose:
@@ -1093,7 +1093,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    else: 
             return filtered_df
 
-    def __evaluate_query(self, query, df, auth_map):
+    def __evaluate_query(self, query, df, auth_map):
         if isinstance(query, BunnyFilter):
             ffunc = self.filter_funcs[query.filter_type]
             result = ffunc(df, query.filter_value, auth_map)
@@ -1107,7 +1107,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-    def apply_filter(self, df, filters, filter_in_core=True, do_author_match=True):
+    def apply_filter(self, df, filters, filter_in_core=True, do_author_match=True):
         if 'eid' not in df and do_author_match:
             do_author_match = False
             warnings.warn('Input DataFrame does not contain Scopus information', RuntimeWarning)
@@ -1119,7 +1119,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-    def add_to_penguin(self, source, path):
+    def add_to_penguin(self, source, path):
         """
         Get the downloaded papers for a given `source` collection from Penguin, represented as a 
         fixed memory-size Bloom filter. This can then be given to iPenguin child objects to prevent
@@ -1143,7 +1143,7 @@ 
    Source code for TELF.applications.Bunny.bunny
    
 [docs]
-    def get_penguin_cache(self, source, max_items=1.25, false_positive_rate=0.001):
+    def get_penguin_cache(self, source, max_items=1.25, false_positive_rate=0.001):
         """
         Get the downloaded papers for a given `source` collection from Penguin, represented as a 
         fixed memory-size Bloom filter. This can then be given to iPenguin child objects to prevent
@@ -1187,11 +1187,11 @@ 
    Source code for TELF.applications.Bunny.bunny
    @property
-    def s2_key(self):
+    def s2_key(self):
         return self._s2_key
 
     @s2_key.setter
-    def s2_key(self, s2_key):
+    def s2_key(self, s2_key):
         if s2_key is not None:
             try:
                 ip = SemanticScholar(key=s2_key)
@@ -1200,11 +1200,11 @@ 
    Source code for TELF.applications.Bunny.bunny
    self._s2_key = s2_key
 
     @property
-    def penguin_settings(self):
+    def penguin_settings(self):
         return self._penguin_settings
 
     @penguin_settings.setter
-    def penguin_settings(self, penguin_settings):
+    def penguin_settings(self, penguin_settings):
         if penguin_settings is not None:
             if not isinstance(penguin_settings, dict):
                 raise TypeError(f'Unsupported type {type(penguin_settings)} for `penguin_settings`. Expected dict')
diff --git a/docs/_modules/TELF/applications/Cheetah/cheetah.html b/docs/_modules/TELF/applications/Cheetah/cheetah.html
index 728988ad..72f3437b 100644
--- a/docs/_modules/TELF/applications/Cheetah/cheetah.html
+++ b/docs/_modules/TELF/applications/Cheetah/cheetah.html
@@ -7,7 +7,7 @@
   
     
     
-    TELF.applications.Cheetah.cheetah — TELF 0.0.41 documentation
+    TELF.applications.Cheetah.cheetah — TELF 0.0.42 documentation
   
   
   
@@ -29,7 +29,7 @@
   
 
 
-    
+    
     
     
   
@@ -39,7 +39,7 @@
   
 
 
-    
+    
     
     
     
@@ -125,7 +125,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
@@ -330,21 +330,21 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    @author: maksimekineren
 """
 
-import os
-import sys
-import ast
-import time
-import pickle
-import warnings
-import pandas as pd
-from tqdm import tqdm
-from collections import Counter, defaultdict
-from typing import Union
+import os
+import sys
+import ast
+import time
+import pickle
+import warnings
+import pandas as pd
+from tqdm import tqdm
+from collections import Counter, defaultdict
+from typing import Union
 
 
 
    
 [docs]
-def add_with_union_of_others(d, s, key):
+def add_with_union_of_others(d, s, key):
     """
     Compute the addition of a set associated with a given key and the union of all other sets in a dictionary.
 
@@ -387,7 +387,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    
 [docs]
-class Cheetah:
+class Cheetah:
 
 
     # keys are fields accepted by Cheetah, values are default column names in DataFrame
@@ -400,7 +400,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    }
 
 
-    def __init__(self, verbose: bool) -> None:
+    def __init__(self, verbose: bool) -> None:
         """ 
         Init an empty Cheetah object
 
@@ -421,7 +421,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    
 [docs]
     @classmethod
-    def find_ngram(cls, text:str, query:list, window_size:int=5, ordered:bool=True) -> bool:
+    def find_ngram(cls, text:str, query:list, window_size:int=5, ordered:bool=True) -> bool:
         """
         Determine if the tokens in the list query are contained within the string text using 
         a sliding window algorithm with a specified window_size. If ordered is True then the 
@@ -475,7 +475,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    
 [docs]
-    def index(self, data:pd.DataFrame, columns:dict=None, index_file:str=None, reindex:bool=False, verbose:bool=True) -> None:
+    def index(self, data:pd.DataFrame, columns:dict=None, index_file:str=None, reindex:bool=False, verbose:bool=True) -> None:
         """
         Creates indices for selected columns in data for Cheetah search. author_ids and affiliations are
         expected to use the their respective SLIC data structures. See an example notebook for a sample
@@ -541,7 +541,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    
 [docs]
-    def search(self, query:list=None, and_search:bool=True, in_title:bool=True, in_abstract:bool=True, 
+    def search(self, query:list=None, and_search:bool=True, in_title:bool=True, in_abstract:bool=True, 
                      save_path:bool=None, author_filter:list=[], affiliation_filter:list=[], 
                      country_filter:list=[], year_filter:list=[], ngram_window_size:int=5, 
                      ngram_ordered:bool=True, do_results_table=False, link_search=False) -> pd.DataFrame:
@@ -706,7 +706,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    def _create_results_table(self, filter_results, query_results):
+    def _create_results_table(self, filter_results, query_results):
         """
         Creates a DataFrame that explains why and how documents were selected by Cheetah filters
 
@@ -765,7 +765,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return results_table_df
             
             
-    def _filter_search(self, col:str, filters:list, indexing:dict) -> set:
+    def _filter_search(self, col:str, filters:list, indexing:dict) -> set:
         """
         Searches the index for the given filters in the specified column.
 
@@ -797,7 +797,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return set.union(*filter_indices)
 
 
-    def _query_search(self, queries:list, in_abstract:bool, in_title:bool, link_search:bool) -> dict:
+    def _query_search(self, queries:list, in_abstract:bool, in_title:bool, link_search:bool) -> dict:
         """
         Searches the text data for the given queries in the specified columns.
 
@@ -867,7 +867,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    def _inclusion_search(self, text_index, q_indices, positives, negatives):
+    def _inclusion_search(self, text_index, q_indices, positives, negatives):
         assert text_index in ('title', 'abstract'), f'Invalid text_index {text_index!r}'
         if text_index == 'title':
             index = self.title_index
@@ -886,7 +886,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return q_indices
     
     
-    def _ngram_check(self, all_indices, all_queries, in_abstract, in_title) -> list:
+    def _ngram_check(self, all_indices, all_queries, in_abstract, in_title) -> list:
         """
         Check for the occurrence of n-grams in a text or document based on some criteria.
 
@@ -931,7 +931,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return all_indices
 
 
-    def _ngram_check_helper(self, index_data:pd.DataFrame, query:str, col:str) -> set:
+    def _ngram_check_helper(self, index_data:pd.DataFrame, query:str, col:str) -> set:
         """
         Perform an n-gram check on the specified DataFrame, query string, and column name.
 
@@ -964,7 +964,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return to_remove
 
 
-    def _index_text(self, data:dict, column:str) -> dict:
+    def _index_text(self, data:dict, column:str) -> dict:
         """
         Indexes the text data in the specified column of a dictionary of records.
 
@@ -1013,7 +1013,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return index_map
 
 
-    def _index_year(self, data:dict) -> dict:
+    def _index_year(self, data:dict) -> dict:
         """
         Indexes the year data of a dictionary of records.
 
@@ -1054,7 +1054,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return year_index
 
     
-    def _index_author(self, data:dict) -> dict:
+    def _index_author(self, data:dict) -> dict:
         """
         Indexes the author data of a dictionary of records.
 
@@ -1102,7 +1102,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return author_index
     
     
-    def _index_affiliation_country(self, data:dict) -> tuple:
+    def _index_affiliation_country(self, data:dict) -> tuple:
         """
         Indexes the country affiliation data of a dictionary of records.
 
@@ -1182,7 +1182,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return affiliation_index, country_index
         
 
-    def _index_data(self, data:dict):
+    def _index_data(self, data:dict):
         """
  
         Parameters
@@ -1231,7 +1231,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    # GETTERS 
 
     @property 
-    def ngram_window_size(self) -> int:
+    def ngram_window_size(self) -> int:
         """
         Get the numeric size of the ngram window. 
         
@@ -1247,7 +1247,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return self._ngram_window_size
 
     @property
-    def ngram_ordered(self) -> bool:
+    def ngram_ordered(self) -> bool:
         """
         Get the status of ngram_ordered.
 
@@ -1263,7 +1263,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return self._ngram_ordered
 
     @property
-    def query(self) -> Union[list, str, None]:
+    def query(self) -> Union[list, str, None]:
         """
         Get the last query of the object.
 
@@ -1279,7 +1279,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    return self._query
 
     @property
-    def columns(self) -> dict:
+    def columns(self) -> dict:
         """
         Retrieve the columns.
 
@@ -1297,7 +1297,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    # SETTERS
 
     @ngram_window_size.setter
-    def ngram_window_size(self, ngram_window_size:int) -> None:
+    def ngram_window_size(self, ngram_window_size:int) -> None:
         """
         Set or update the size of the n-gram window of the object.
 
@@ -1318,7 +1318,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    self._ngram_window_size = ngram_window_size
 
     @ngram_ordered.setter
-    def ngram_ordered(self, ngram_ordered:bool) -> None:
+    def ngram_ordered(self, ngram_ordered:bool) -> None:
         """
         Set or update a flag that controls whether n-grams should be ordered or not.
 
@@ -1337,7 +1337,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    @query.setter
-    def query(self, query: Union[list, str, dict, None]) -> None:
+    def query(self, query: Union[list, str, dict, None]) -> None:
         """
         Set or update the query of the object.
 
@@ -1350,8 +1350,8 @@ 
    Source code for TELF.applications.Cheetah.cheetah
            -------
         None
         """
-        def process_entry(entry):
-            def validate_dependent_terms(terms):
+        def process_entry(entry):
+            def validate_dependent_terms(terms):
                 for term in terms:
                     if len(term.split()) > 1:
                         raise ValueError(f"Expected single token in dependent term, but got {term!r}")
@@ -1401,7 +1401,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    @columns.setter
-    def columns(self, columns:dict) -> None:
+    def columns(self, columns:dict) -> None:
         """
         Set or update the columns of a data structure or object.
         
@@ -1439,7 +1439,7 @@ 
    Source code for TELF.applications.Cheetah.cheetah
    # UTIL
 
     @classmethod
-    def __error_unexpected_type(cls, expected_type, var) -> ValueError:
+    def __error_unexpected_type(cls, expected_type, var) -> ValueError:
         """
         Raises an ValueError exception
 
diff --git a/docs/_modules/TELF/applications/Penguin/penguin.html b/docs/_modules/TELF/applications/Penguin/penguin.html
index 73462228..2eb03c04 100644
--- a/docs/_modules/TELF/applications/Penguin/penguin.html
+++ b/docs/_modules/TELF/applications/Penguin/penguin.html
@@ -7,7 +7,7 @@
   
     
     
-    TELF.applications.Penguin.penguin — TELF 0.0.41 documentation
+    TELF.applications.Penguin.penguin — TELF 0.0.42 documentation
   
   
   
@@ -29,7 +29,7 @@
   
 
 
-    
+    
     
     
   
@@ -39,7 +39,7 @@
   
 
 
-    
+    
     
     
     
@@ -125,7 +125,7 @@
   
   
   
-    
    TELF 0.0.41 documentation
    
+    
    TELF 0.0.42 documentation
    
   
 
    
         
    
@@ -322,27 +322,28 @@ 
    
                 
    
                   
   
    Source code for TELF.applications.Penguin.penguin
    -import re
-import sys
-import pathlib
-import warnings
-import pandas as pd
-from tqdm import tqdm
-from rbloom import Bloom
-from joblib import Parallel, delayed
-
-from pymongo import MongoClient, ASCENDING
-from pymongo.errors import ConnectionFailure
-
-from ...helpers.host import verify_n_jobs
-from ...helpers.data_structures import verify_attributes, transform_dict
-from .utils import create_text_query
-from .crocodile import process_scopus_json, process_scopus_xml, process_s2_json 
-from .crocodile import form_scopus_df, form_s2_df, form_df
+import re
+import sys
+import pathlib
+import warnings
+import pandas as pd
+from tqdm import tqdm
+from rbloom import Bloom
+from joblib import Parallel, delayed
+from typing import Sequence
+
+from pymongo import MongoClient, ASCENDING
+from pymongo.errors import ConnectionFailure
+
+from ...helpers.host import verify_n_jobs
+from ...helpers.data_structures import verify_attributes, transform_dict
+from .utils import create_text_query
+from .crocodile import process_scopus_json, process_scopus_xml, process_s2_json 
+from .crocodile import form_scopus_df, form_s2_df, form_df
 
 
    
 [docs]
-class Penguin:
+class Penguin:
     """
     A handler class for managing connections and operations with a documents MongoDB database.
 
@@ -377,7 +378,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    }
 
     
-    def __init__(self, uri, db_name, username=None, password=None, n_jobs=-1, verbose=False):
+    def __init__(self, uri, db_name, username=None, password=None, n_jobs=-1, verbose=False):
         """
         Initializes the Penguin instance with the specified URI, database name, and optional settings.
 
@@ -429,7 +430,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    # 1. Connecting to the Penguin Database
         
         
-    def _connect(self):
+    def _connect(self):
         """
         Establishes a connection to the MongoDB database and validates its structure.
 
@@ -476,7 +477,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    raise(e)
 
             
-    def _validate_database(self, required_collections):
+    def _validate_database(self, required_collections):
         """
         Validates the presence of required collections in the connected MongoDB database.
 
@@ -524,7 +525,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    
 [docs]
-    def add_many_documents(self, directory, source, overwrite=True, n_jobs=None):
+    def add_many_documents(self, directory, source, overwrite=True, n_jobs=None):
         """
         Processes a directory of document files that need to be added to the database. The
         function can add both Scopus and S2 documents depending on the `source` agument. 
@@ -560,7 +561,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    
 [docs]
-    def add_single_document(self, file_path, source, overwrite=True):
+    def add_single_document(self, file_path, source, overwrite=True):
         """
         Processes a single data file and adds / updates its content in the database.
         
@@ -601,7 +602,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    def _file_path_generator(self, directory, ext):
+    def _file_path_generator(self, directory, ext):
         """
         Generator function that yields paths to files with certain extensions in 
         the given directory.
@@ -631,7 +632,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    yield file_path
             
             
-    def _add_single_s2_file(self, file_path, overwrite=True):
+    def _add_single_s2_file(self, file_path, overwrite=True):
         """
         Processes a single S2 data file for addition 
 
@@ -661,7 +662,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    if file_path.suffix == '.json':
             data = process_s2_json(file_path, output_dir=None)
         else:
-            raise ValueError(f'S2 `file_path` has an unsupported file extension: {source!r}')
+            raise ValueError(f'S2 `file_path` has an unsupported file extension: {file_path!r}')
         
         collection = self.db[self.S2_COL]
         id_attr = self.s2_attributes['id']
@@ -675,7 +676,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    pass
                 
                 
-    def _add_single_scopus_file(self, file_path, overwrite=True):
+    def _add_single_scopus_file(self, file_path, overwrite=True):
         """
         Processes a single Scopus data file for addition 
 
@@ -707,7 +708,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    elif file_path.suffix == '.json':
             data = process_scopus_json(file_path, output_dir=None)
         else:
-            raise ValueError(f'Scopus `file_path` has an unsupported file extension: {source!r}')
+            raise ValueError(f'Scopus `file_path` has an unsupported file extension: {file_path!r}')
         
         collection = self.db[self.SCOPUS_COL]
         id_attr = self.scopus_attributes['id']
@@ -726,7 +727,7 @@ 
    Source code for TELF.applications.Penguin.penguin
    
 [docs]
-    def count_documents(self):
+    def count_documents(self):
         """
         Return the number of documents in the Scopus and S2.
     
@@ -755,7 +756,7 @@ 
    Source code for TELF.applications.Penguin.penguin