Updates & debugs (#20)

CodyCBakerPhD · pre-commit-ci[bot] · web-flow · commit a0d21358359e · 2025-07-10T01:43:53.000-04:00
* feat: add compatability for new structuree * feat: update base branch * feat: add pre-commit * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * feat: add pre-commit --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,2 @@
 # PyCharm
-.idea/*
+.idea/*
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,25 @@
+default_install_hook_types:
+  - pre-commit
+  - commit-msg
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+
+- repo: https://github.com/codespell-project/codespell
+  rev: v2.4.1
+  hooks:
+  - id: codespell
+    additional_dependencies:
+    - tomli
+
+- repo: https://github.com/compilerla/conventional-pre-commit
+  rev: v4.2.0
+  hooks:
+    - id: conventional-pre-commit
+      stages: [commit-msg]
+      args: [--verbose]
diff --git a/index.html b/index.html
@@ -6,6 +6,7 @@
 
     <title>Dandiset Access Summaries</title>
 
+    <script src="https://cdn.jsdelivr.net/npm/js-yaml@4.1.0/dist/js-yaml.min.js"></script>
     <script src="https://cdn.plot.ly/plotly-3.0.1.min.js" charset="utf-8"></script>
     <script src="errors.js" defer></script>
     <script src="plots.js" defer></script>
diff --git a/plots.js b/plots.js
@@ -6,8 +6,8 @@ const BASE_URL = "https://raw.githubusercontent.com/dandi/access-summaries/main"
 const BASE_TSV_URL = `${BASE_URL}/content/summaries`;
 
 const ARCHIVE_TOTALS_URL = `${BASE_URL}/content/archive_totals.json`;
-const ALL_DANDISET_TOTALS_URL = `${BASE_URL}/content/all_dandiset_totals.json`;
-const REGION_CODES_TO_LATITUDE_LONGITUDE_URL = `${BASE_URL}/content/region_codes_to_coordinates.json`;
+const ALL_DANDISET_TOTALS_URL = `${BASE_URL}/content/totals.json`;
+const REGION_CODES_TO_LATITUDE_LONGITUDE_URL = `${BASE_URL}/content/region_codes_to_coordinates.yaml`;
 
 let REGION_CODES_TO_LATITUDE_LONGITUDE = {};
 let ALL_DANDISET_TOTALS = {};
@@ -28,11 +28,11 @@ window.addEventListener("load", () => {
     if (logScaleCheckbox) {
         logScaleCheckbox.addEventListener("change", function() {
             USE_LOG_SCALE = this.checked;
-            
+
             // Get the current dandiset ID
             const dandiset_selector = document.getElementById("dandiset_selector");
             const selected_dandiset = dandiset_selector.value;
-            
+
             // Reload plots with the current dandiset ID
             load_over_time_plot(selected_dandiset);
             load_per_asset_histogram(selected_dandiset);
@@ -112,15 +112,15 @@ function resizePlots() {
 fetch(REGION_CODES_TO_LATITUDE_LONGITUDE_URL)
     .then((response) => {
         if (!response.ok) {
-            throw new Error(`Failed to fetch JSON file: ${response.statusText}`);
+            throw new Error(`Failed to fetch YAML file: ${response.statusText}`);
         }
-        return response.json();
+        return response.text();
     })
     .then((data) => {
-        REGION_CODES_TO_LATITUDE_LONGITUDE = data;
+        REGION_CODES_TO_LATITUDE_LONGITUDE = jsyaml.load(data);
     })
     .catch((error) => {
-        console.error("Error loading JSON file:", error);
+        console.error("Error loading YAML file:", error);
     });
 
 fetch(ARCHIVE_TOTALS_URL)
@@ -153,7 +153,20 @@ fetch(ALL_DANDISET_TOTALS_URL)
     })
     .then((all_dandiset_totals_text) => {
         Object.assign(ALL_DANDISET_TOTALS, JSON.parse(all_dandiset_totals_text));
-        const dandiset_ids = Object.keys(ALL_DANDISET_TOTALS);
+        let dandiset_ids = Object.keys(ALL_DANDISET_TOTALS);
+        dandiset_ids.sort((a, b) => {
+            if (a === "archive") return -1;
+            if (b === "archive") return 1;
+            // Compare as numbers if both are numeric
+            const aNum = Number(a);
+            const bNum = Number(b);
+            if (!isNaN(aNum) && !isNaN(bNum)) {
+                return aNum - bNum;
+            }
+            // Fallback to string comparison
+            return a.localeCompare(b);
+        });
+
         const selector = document.getElementById("dandiset_selector");
 
         if (!selector) {
@@ -208,7 +221,7 @@ function update_totals(dandiset_id) {
         const footnote = document.createElement("div");
         footnote.style.fontSize = "0.5em";
         footnote.style.marginTop = "7px";
-        footnote.innerHTML = "<sup>*</sup> These values are only estimates for publically released datasets and are subject to change as additional information becomes available.";
+        footnote.innerHTML = "<sup>*</sup> These values are only estimates for publicly released datasets and are subject to change as additional information becomes available.";
         totals_element.appendChild(footnote);
     } catch (error) {
         console.error("Error:", error);
@@ -221,13 +234,7 @@ function update_totals(dandiset_id) {
 // Function to fetch and render the over time for a given Dandiset ID
 function load_over_time_plot(dandiset_id) {
     const plot_element_id = "over_time_plot";
-    let by_day_summary_tsv_url;
-
-    if (dandiset_id === "archive") {
-        by_day_summary_tsv_url = `${BASE_TSV_URL}/archive_summary_by_day.tsv`;
-    } else {
-        by_day_summary_tsv_url = `${BASE_TSV_URL}/${dandiset_id}/dandiset_summary_by_day.tsv`;
-    }
+    let by_day_summary_tsv_url = `${BASE_TSV_URL}/${dandiset_id}/by_day.tsv`;
 
     fetch(by_day_summary_tsv_url)
         .then((response) => {
@@ -261,8 +268,9 @@ function load_over_time_plot(dandiset_id) {
 
             const plot_info = [
                 {
-                    type: "scatter",
-                    mode: "lines+markers",
+                    //type: "scatter",
+                    //mode: "lines+markers",
+                    type: "bar",
                     x: dates, // Use raw dates for proper alignment
                     y: plot_data,
                     text: dates.map((date, index) => `${date}<br>${human_readable_bytes_sent[index]}`),
@@ -310,7 +318,7 @@ function load_over_time_plot(dandiset_id) {
 // Function to fetch and render histogram over asset IDs
 function load_per_asset_histogram(dandiset_id) {
     const plot_element_id = "per_asset_histogram";
-    let by_day_summary_tsv_url = "";
+    let by_asset_summary_tsv_url = "";
 
     // Suppress div element content if 'archive' is selected
     if (dandiset_id === "archive") {
@@ -320,10 +328,10 @@ function load_per_asset_histogram(dandiset_id) {
         }
         return "";
     } else {
-        by_day_summary_tsv_url = `${BASE_TSV_URL}/${dandiset_id}/dandiset_summary_by_asset.tsv`;
+        by_asset_summary_tsv_url = `${BASE_TSV_URL}/${dandiset_id}/by_asset.tsv`;
     }
 
-    fetch(by_day_summary_tsv_url)
+    fetch(by_asset_summary_tsv_url)
         .then((response) => {
             if (!response.ok) {
                 throw new Error(`Failed to fetch TSV file: ${response.statusText}`);
@@ -346,23 +354,25 @@ function load_per_asset_histogram(dandiset_id) {
                     throw new Error("Currently only supports NWB files.");
                 }
 
-                // TODO: this was a heuristic idea for shortening the asset names
-                // const subject_and_session = filename.split("_");
-                // const subject = subject_and_session.at(0).split("-").slice(1).join("-");
-                // const session = subject_and_sessions.slice(1).split("-").slice(1).join("-");
-                // return `${subject} ${session}`;
-
                 return filename;
             });
             const bytes_sent = data.map((row) => parseInt(row[1], 10));
-            const human_readable_bytes_sent = bytes_sent.map((bytes) => format_bytes(bytes));
 
+            // Sort asset_names and bytes_sent in descending order by bytes_sent
+            const combined = asset_names.map((name, idx) => ({ name, bytes: bytes_sent[idx] }));
+            combined.sort((a, b) => b.bytes - a.bytes);
+
+            const sorted_asset_names = combined.map(item => item.name);
+            const sorted_bytes_sent = combined.map(item => item.bytes);
+            const human_readable_bytes_sent = sorted_bytes_sent.map((bytes) => format_bytes(bytes));
+
+            // Use sorted arrays in the plot
             const plot_data = [
                 {
                     type: "bar",
-                    x: asset_names,
-                    y: bytes_sent,
-                    text: asset_names.map((name, index) => `${name}<br>${human_readable_bytes_sent[index]}`),
+                    x: sorted_asset_names,
+                    y: sorted_bytes_sent,
+                    text: sorted_asset_names.map((name, index) => `${name}<br>${human_readable_bytes_sent[index]}`),
                     textposition: "none",
                     hoverinfo: "text",
                 }
@@ -375,14 +385,10 @@ function load_per_asset_histogram(dandiset_id) {
                 },
                 xaxis: {
                     title: {
-                        text: "Asset Name",
+                        text: "(hover over an entry for asset names)",
                         font: { size: 16 }
                     },
                     showticklabels: false,
-                    // TODO: ticks are currently too long to fit since heuristic is not working well
-                    // tickangle: -45,
-                    // tickfont: { size: 10 },
-                    // automargin: true,
                 },
                 yaxis: {
                     title: {
@@ -403,21 +409,17 @@ function load_per_asset_histogram(dandiset_id) {
             console.error("Error:", error);
             const plot_element = document.getElementById(plot_element_id);
             if (plot_element) {
-                plot_element.innerText = "Failed to load data for per asset (current supports NWB datasets only).";
+                while (plot_element.firstChild) {
+                    plot_element.removeChild(plot_element.firstChild);
+                }
             }
         });
 }
 
 // Function to fetch and render heatmap over geography
 function load_geographic_heatmap(dandiset_id) {
     const plot_element_id = "geography_heatmap";
-    let by_region_summary_tsv_url;
-
-    if (dandiset_id === "archive") {
-        by_region_summary_tsv_url = `${BASE_TSV_URL}/archive_summary_by_region.tsv`;
-    } else {
-        by_region_summary_tsv_url = `${BASE_TSV_URL}/${dandiset_id}/dandiset_summary_by_region.tsv`;
-    }
+    let by_region_summary_tsv_url = `${BASE_TSV_URL}/${dandiset_id}/by_region.tsv`;
 
     if (!REGION_CODES_TO_LATITUDE_LONGITUDE) {
         console.error("Error:", error);

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`# PyCharm`
`2`		`-.idea/*`
	`2`	`+.idea/*`