Skip to content

Commit

Permalink
Merge pull request #4 from Aquaveo/publications_lazy_loading
Browse files Browse the repository at this point in the history 
Publications lazy loading patch
  • Loading branch information
@romer8
romer8 authored Mar 25, 2024
2 parents bb97f75 + 8225ddb commit e727f8f
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 448 deletions.
243 changes: 39 additions & 204 deletions zotero_publications_app/static/js/publications.js
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,3 @@
// if local publications is bigger than one, then used as an start index to fetch new publications.


// var startLoop = totalLocalPublications > 1 ? totalLocalPublications : 0
// var startLoop = 0
let LIMIT = 10;
let publicationsData = null

let publicationsHtmlElement = document.getElementById('publications-html')
Expand Down Expand Up @@ -40,16 +34,42 @@ function appendArrayValues(obj1, obj2) {
return result;
}

const inititialFetchPublications = (start) => {
const parseDataResponse = (data) =>{
// console.log(data)
loadingHtmlElement.classList.remove('hidden')
if(!publicationsData){
publicationsData = data
}
else{
publicationsData = appendArrayValues(publicationsData, data)
}

let publicationsHTML = ``

for (let [key, value] of Object.entries(publicationsData).reverse()) {

//add the year title
if (key === '1300'){
key = 'More Publications'
}

publicationsHTML += `<h2 class="year-style">${key}</h2>`
//add the publications
value.forEach((publication) => {
publicationsHTML += publication
})
}
publicationsHtmlElement.innerHTML = publicationsHTML;
}

const inititialFetchPublications = () => {
// Update the URL if your setup is different or if you are using a production server
let newRequestData = requestData
newRequestData['start'] = start
newRequestData['limit'] = LIMIT
// newRequestData['isRemote'] = total_publications > totalLocalPublications

newRequestData['totalLocalPublications'] = totalLocalPublications
newRequestData['totalRemotePublications'] = total_publications

newRequestData['instanceID'] = instanceID

return fetch(init_pubs_url, {
method: 'POST',
headers: {
Expand All @@ -67,33 +87,8 @@ const inititialFetchPublications = (start) => {
})

.then(data => {
console.log(data)
loadingHtmlElement.classList.remove('hidden')
if(!publicationsData){
publicationsData = data
}
else{
publicationsData = appendArrayValues(publicationsData, data)
}

let publicationsHTML = ``
parseDataResponse(data)

for (let [key, value] of Object.entries(publicationsData).reverse()) {

//add the year title
if (key === '1300'){
key = 'More Publications'
}

publicationsHTML += `<h2 class="year-style">${key}</h2>`
//add the publications
value.forEach((publication) => {
publicationsHTML += publication
})
}
publicationsHtmlElement.innerHTML = publicationsHTML;

// For example, you could iterate over the data and append it to an element in your HTML
})
.catch(error => {
console.error('There was a problem with the fetch operation:', error);
Expand All @@ -103,11 +98,10 @@ const inititialFetchPublications = (start) => {



const fetchPublications = (start) => {
const fetchPublications = () => {
// Update the URL if your setup is different or if you are using a production server
let newRequestData = requestData
newRequestData['start'] = start
newRequestData['limit'] = LIMIT

// newRequestData['isRemote'] = total_publications > totalLocalPublications
newRequestData['totalLocalPublications'] = totalLocalPublications
newRequestData['totalRemotePublications'] = total_publications
Expand All @@ -130,33 +124,8 @@ const fetchPublications = (start) => {
})

.then(data => {
console.log(data)
loadingHtmlElement.classList.remove('hidden')
if(!publicationsData){
publicationsData = data
}
else{
publicationsData = appendArrayValues(publicationsData, data)
}

let publicationsHTML = ``

for (let [key, value] of Object.entries(publicationsData).reverse()) {

//add the year title
if (key === '1300'){
key = 'More Publications'
}

publicationsHTML += `<h2 class="year-style">${key}</h2>`
//add the publications
value.forEach((publication) => {
publicationsHTML += publication
})
}
publicationsHtmlElement.innerHTML = publicationsHTML;

// For example, you could iterate over the data and append it to an element in your HTML
// console.log(data)
parseDataResponse(data);
})
.catch(error => {
console.error('There was a problem with the fetch operation:', error);
Expand Down Expand Up @@ -214,75 +183,17 @@ create_placeholders();

// get the total number of publications in the library
let total_publications = await get_items_counts();
console.log(total_publications)


const fetchAndProcessPublicationsSequentially = async () => {

// Start the loop based on the total number of local publications
let startLoop = totalLocalPublications > 1 ? totalLocalPublications : 0;

let differencePublications = total_publications - totalLocalPublications;

// if the difference is negative, it means that publications were deleted, so start with index 0
startLoop = differencePublications < 0 ? 0 : startLoop;

console.log("Difference in publications:", differencePublications);

if (totalLocalPublications == 0){
let iterations = differencePublications > 0 ? Math.ceil(differencePublications / LIMIT) : Math.ceil(total_publications / LIMIT);

console.log("Number of iterations:", iterations);


for (let i = 0; i < iterations; i++) {
try {
await inititialFetchPublications(startLoop);
startLoop += LIMIT;
} catch (error) {
console.error("Error with one of the fetch operations:", error);
// Optionally break or continue based on your error handling preferences
break;
}
}
await inititialFetchPublications();

}
else{
LIMIT = Math.abs(differencePublications) //only search for the most recent items added to the library
await fetchPublications(startLoop);
await fetchPublications();
}


// // if new publications were added or deleted, fetch them
// if (differencePublications > 0) {
// // Adjust the loop based on whether you need to add new ones or refetch all
// let iterations = differencePublications > 0 ? Math.ceil(differencePublications / LIMIT) : Math.ceil(total_publications / LIMIT);

// console.log("Number of iterations:", iterations);


// for (let i = 0; i < iterations; i++) {
// try {
// await fetchPublications(startLoop);
// startLoop += LIMIT;
// } catch (error) {
// console.error("Error with one of the fetch operations:", error);
// // Optionally break or continue based on your error handling preferences
// break;
// }
// }
// }
// if(differencePublications < 0){
// // let iterations = Math.ceil(total_publications / LIMIT);

// LIMIT = Math.abs(differencePublications) //only search for the most recent items added to the library
// await fetchPublications(startLoop);

// }
// if(differencePublications==0) {
// console.log("No changes, retrieving from database or using cached data");
// await fetchPublications(startLoop);
// }

// Hide the loading HTML element once all operations are complete
loadingHtmlElement.classList.add('hidden');
};
Expand All @@ -294,79 +205,3 @@ fetchAndProcessPublicationsSequentially().then(() => {
console.error("An error occurred during the publication fetching process:", error);
});



// //second iteration
// if(total_publications > 0){
// let fetchPromises = [];
// let differencePublications = total_publications - totalLocalPublications;

// //making batch requests
// console.log("difference in publications",differencePublications)

// // publications were added to the remote library, in this case just add the new ones
// if (differencePublications > 0){
// console.log("new applications added")
// for (var i = 0; i < Math.ceil(differencePublications/LIMIT); i++) {
// // Collect promises returned by fetchPublications
// fetchPromises.push(fetchPublications(startLoop));
// startLoop += LIMIT;
// }
// }
// // publications were deleted in the library, in this case just add all of them again
// else if(differencePublications < 0){
// console.log("applications deleted")
// startLoop = 0 //reset the startLoop, we are going to fetch all the publications again
// for (var i = 0; i < Math.ceil(total_publications/LIMIT); i++) {
// // Collect promises returned by fetchPublications
// fetchPromises.push(fetchPublications(startLoop));
// startLoop += LIMIT;
// }
// }
// // retrieving from database if they are the same.
// else{
// console.log("no changes, retrieving from database")
// fetchPromises.push(fetchPublications(startLoop));
// }

// Promise.all(fetchPromises)
// .then(() => {
// loadingHtmlElement.classList.add('hidden');
// })
// .catch(error => {
// console.error("Error with one of the fetch operations:", error);
// });

// }









//first edit

// if(total_publications > 0){
// let fetchPromises = [];
// let differencePublications = total_publications - totalLocalPublications;

// for (var i = 0; i < Math.ceil(total_publications/LIMIT); i++) {
// // Collect promises returned by fetchPublications
// if(total_publications > startLoop){
// fetchPromises.push(fetchPublications(startLoop));
// startLoop += LIMIT;
// }
// }

// Promise.all(fetchPromises)
// .then(() => {
// loadingHtmlElement.classList.add('hidden');
// })
// .catch(error => {
// console.error("Error with one of the fetch operations:", error);
// });

// }
2 changes: 1 addition & 1 deletion zotero_publications_app/templates/zotero-publications.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

// get the local publications count from the plugin instance
let LocalPublicationsObject = JSON.parse(JSON.stringify({{ instance.publications | safe }}));
console.log(LocalPublicationsObject);

let totalLocalPublications = Object.values(LocalPublicationsObject).reduce((sum, currentValue) => {
return sum + currentValue.length;
}, 0);
Expand Down
46 changes: 16 additions & 30 deletions zotero_publications_app/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ def get_merged_publications_count(publications):

def get_remote_latest_version(instance):
zot = zotero.Zotero(
instance.get("library_id"),
instance.get("library_type"),
instance.get("api_key"),
instance.library_id,
instance.library_type,
instance.api_key,
)

latest_version = zot.last_modified_version()
Expand All @@ -36,12 +36,12 @@ def get_publications(instance, params):
# Initialize a dictionary to store publications by year
publications_by_year = {}
zot = zotero.Zotero(
instance.get("library_id"),
instance.get("library_type"),
instance.get("api_key"),
instance.library_id,
instance.library_type,
instance.api_key,
)
if instance.get("collection_id"):
items = zot.collection_items(instance.get("collection_id"), **params)
if instance.collection_id:
items = zot.collection_items(instance.collection_id, **params)
else:

# logging.warning(number_items)
Expand All @@ -67,19 +67,11 @@ def get_publications(instance, params):

def get_trashed_publications_ids(instance, params):
zot = zotero.Zotero(
instance.get("library_id"),
instance.get("library_type"),
instance.get("api_key"),
instance.library_id,
instance.library_type,
instance.api_key,
)

# params = {
# "sort": "dateAdded", # get the latest added
# "direction": "desc", # get in asc order, so we can compare the latest added and indexes
# "linkwrap": 1,
# "start": 0,
# "limit": 100,
# }

trashed_items_ids = []
items_trashed = zot.trash(**params)

Expand All @@ -92,12 +84,12 @@ def get_trashed_publications_ids(instance, params):
def get_deleted_publications_ids(instance):
# Initialize a dictionary to store publications by year
zot = zotero.Zotero(
instance.get("library_id"),
instance.get("library_type"),
instance.get("api_key"),
instance.library_id,
instance.library_type,
instance.api_key,
)
params = {
"since": instance.get("current_local_version"),
"since": instance.local_remote_version,
}

deleted_items_ids = []
Expand All @@ -121,13 +113,7 @@ def get_all_the_removed_publications_ids(instance, params):
return trashed_items_ids + deleted_items_ids


def delete_publications_from_local_instance_by_id_list(instanceId, id_list):

local_instance = ZoteroPublications.objects.get(id=instanceId)
# for key, value in local_instance.publications.items():
# local_instance.publications[key] = [
# item for item in value if item.get("key", "") not in id_list
# ]
def delete_publications_from_local_instance_by_id_list(local_instance, id_list):

for key, list_of_dicts in local_instance.publications.items():
# Filter out dictionaries whose keys are in the ids_to_delete list
Expand Down
Loading

0 comments on commit e727f8f

Please sign in to comment.