Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pagination & Filters #154

Merged
merged 30 commits into from
Mar 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
656e0b6
Update friend suggestions endpoint to use paginator
Piterson25 Mar 15, 2024
c70bee1
Update friend suggestions endpoint to use query params
Piterson25 Mar 15, 2024
bc7efec
Add tests for friend suggestions
Piterson25 Mar 15, 2024
6726bce
Merge branch 'dev' into pagination
Piterson25 Mar 16, 2024
e681ee7
Fix search test
Piterson25 Mar 16, 2024
f69cf1e
Add search filter by country name
Piterson25 Mar 23, 2024
337dd9d
Update search to use search term with country
Piterson25 Mar 23, 2024
dd1dc4a
Update friend suggestions endpoint to use pagination as optional
Piterson25 Mar 24, 2024
06f344d
Update friend suggestions tests
Piterson25 Mar 24, 2024
0cd5393
Add search tests
Piterson25 Mar 24, 2024
d21f343
Add pagination and tests to friends endpoint
Piterson25 Mar 24, 2024
ee6b36d
Add filter option to search
Karol-2 Mar 24, 2024
300c80b
Change styling of filters
Karol-2 Mar 24, 2024
b1f8c25
Fix errors with filters
Karol-2 Mar 24, 2024
6e9112c
Add PaginatorV2
Karol-2 Mar 24, 2024
177ff75
Add error handling
Karol-2 Mar 24, 2024
39bff9b
Add allUsersSize to search endpoint
Piterson25 Mar 25, 2024
9475148
Update friends and friend suggestions endpoint
Piterson25 Mar 25, 2024
6bba980
Fix page totalPage numbers
Piterson25 Mar 25, 2024
b233372
Fix totalPage
Piterson25 Mar 25, 2024
0b1a977
Add PagintorV2 to friends
Karol-2 Mar 25, 2024
90ff6d2
Add PagnatorV2 to suggestions and search
Karol-2 Mar 25, 2024
140374c
Fix endpoints on search
Karol-2 Mar 25, 2024
05ebd11
Fix suggestions
Karol-2 Mar 25, 2024
0ee9488
Fix search error
Karol-2 Mar 25, 2024
066d721
Fix searchPage error
Karol-2 Mar 25, 2024
c8642bf
Fix tests
Piterson25 Mar 25, 2024
0fef38b
Apply formatting
Piterson25 Mar 25, 2024
10fcad0
Fix casting params for page and maxUsersOnPage
Piterson25 Mar 25, 2024
e510812
Merge branch 'pagination' of github.com:Karol-2/Mercury-Project into …
Piterson25 Mar 25, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
993 changes: 837 additions & 156 deletions backend/package-lock.json

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion backend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@
"nodemon": "^3.0.1",
"socket.io": "^4.7.2",
"ts-node": "^10.9.1",
"unidecode": "^0.1.8"
"unidecode": "^0.1.8",
"vite": "^5.2.6"
},
"devDependencies": {
"@types/bcrypt": "^5.0.2",
Expand Down
3 changes: 3 additions & 0 deletions backend/src/misc/roundToInt.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
export default function roundToInt(value: number) {
return Math.ceil(value);
}
4 changes: 3 additions & 1 deletion backend/src/models/Response.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ export interface FriendsResponse {

export interface UsersSearchResponse {
status: "ok";
users: [User, number][];
allUsersSize: number;
totalPage: number;
users: User[];
}

export interface JWTResponse extends OkResponse {
Expand Down
119 changes: 107 additions & 12 deletions backend/src/routes/usersFriendsRoute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,16 @@ import { Router, Request, Response } from "express";
import { Session } from "neo4j-driver";
import driver from "../driver/driver";
import User from "../models/User";
import removeKeys from "../misc/removeKeys";
import roundToInt from "../misc/roundToInt";
import {
OkErrorResponse,
FriendsErrorResponse,
UsersErrorResponse,
} from "../types/userResponse";

const filterUser = (user: User) => removeKeys({ ...user }, ["name_embedding"]);

const friendshipRouter = Router();

async function userExists(
Expand All @@ -31,27 +35,72 @@ async function userExists(

friendshipRouter.get(
"/:userId/friends",
async (req: Request, res: FriendsErrorResponse) => {
async (req: Request, res: Response) => {
try {
const session = driver.session();
const userId = req.params.userId;
const page: number = parseInt((req.query.page as string) || "");
const maxUsersOnPage: number = parseInt(
(req.query.maxUsers as string) || "",
);

const user = await userExists(session, res, userId);
if ("json" in user) {
await session.close();
return res;
}

const friendQuery = await session.run(
const searchRequest = await session.run(
`MATCH (u:User {id: $userId})-[:IS_FRIENDS_WITH]->(f:User)-[:IS_FRIENDS_WITH]->(u)
WITH f ORDER BY f.last_name, f.first_name
RETURN DISTINCT f`,
{ userId },
);
await session.close();

const friends = friendQuery.records.map((f) => f.get("f").properties);
return res.json({ status: "ok", friends });
const allFriends = searchRequest.records.map((f) => {
return filterUser(f.get("f").properties);
});

if (!page && !maxUsersOnPage) {
if (allFriends.length === 0) {
return res.status(404).json({
status: "error",
errors: { users: "No friends found" },
});
}
const totalPage: number = roundToInt(allFriends.length / 5);
return res.status(200).json({
status: "ok",
allUsersSize: allFriends.length,
totalPage: totalPage,
users: allFriends,
});
} else if (!page || !maxUsersOnPage) {
return res.status(400).json({
status: "error",
errors: { params: "Missing or incorrect query params" },
});
}

const friends = allFriends.slice(
(page - 1) * maxUsersOnPage,
page * maxUsersOnPage,
);

if (friends.length === 0) {
return res.status(404).json({
status: "error",
errors: { users: "No friends found with given queries" },
});
}
const totalPage: number = roundToInt(allFriends.length / maxUsersOnPage);
return res.status(200).json({
status: "ok",
allUsersSize: allFriends.length,
totalPage: totalPage,
users: friends,
});
} catch (err) {
console.log("Error:", err);
return res.status(404).json({ status: "error", errors: err as object });
Expand Down Expand Up @@ -91,11 +140,14 @@ friendshipRouter.get(

friendshipRouter.get(
"/:userId/friend-suggestions",
async (req: Request, res: UsersErrorResponse) => {
async (req: Request, res: Response) => {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Response isn't typed so type checking doesn't work

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't remember if it was already in code or I added, but changing the Response to custom would cause to change it everywhere where friends are mentioned, like friend-suggestions or friends endpoints, each to corresponing endpoint. I didn't come to any other conslusion, so if you have any, let me know! 😊

try {
const session: Session = driver.session();
const userId: string = req.params.userId;

const page: number = parseInt((req.query.page as string) || "");
const maxUsersOnPage: number = parseInt(
(req.query.maxUsers as string) || "",
);
const user = await userExists(session, res, userId);
if ("json" in user) {
await session.close();
Expand All @@ -108,15 +160,58 @@ friendshipRouter.get(
RETURN DISTINCT suggested`,
{ userId },
);

const allUsers: User[] = friendSuggestionsQuery.records.map((r) => {
return filterUser(r.get("suggested").properties);
});

await session.close();

const users: User[] = friendSuggestionsQuery.records
.map((record) => record.get("suggested").properties)
.slice(0, 15);
return res.json({ status: "ok", users });
if (!page && !maxUsersOnPage) {
if (allUsers.length === 0) {
return res.status(404).json({
status: "not found",
message: "No users found",
});
}
const totalPage: number = roundToInt(allUsers.length / 5);
return res.status(200).json({
status: "ok",
allUsersSize: allUsers.length,
totalPage: totalPage,
users: allUsers,
});
} else if (!page || !maxUsersOnPage) {
return res.status(400).json({
status: "bad request",
message: "Missing or incorrect query params",
});
}

const users = allUsers.slice(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code would still be slow if a million users registered. It's still taking all the users from the database.
I think it should be done in the database like this: MATCH (u:User) RETURN u LIMIT 16 SKIP n*16. This way the database can cache the result and it should be really fast

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm I never thought of this solution, thanks! I will try to fix this now :)

(page - 1) * maxUsersOnPage,
page * maxUsersOnPage,
);

if (users.length === 0) {
return res.status(404).json({
status: "not found",
message: "No users found with given queries",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All other responses have a different format: { status: "error", errors: { "users": "not found" } }.
In this case the page having no users isn't an error, the page is just empty

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, you're right 😅
I tried to use different response codes, like 400 for bad requests and 404 for not found

});
}

const totalPage: number = roundToInt(allUsers.length / maxUsersOnPage);
return res.status(200).json({
status: "ok",
allUsersSize: allUsers.length,
totalPage: totalPage,
users: users,
});
} catch (err) {
console.log("Error:", err);
return res.status(404).json({ status: "error", errors: err as object });
console.error("Error:", err);
return res
.status(500)
.json({ status: "error", message: "Internal server error" });
}
},
);
Expand Down
139 changes: 84 additions & 55 deletions backend/src/routes/usersRoute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
import usersFriendsRoute from "./usersFriendsRoute";

import removeKeys from "../misc/removeKeys";
import roundToInt from "../misc/roundToInt";
import { ChangePasswordReq } from "../models/ChangePasswordReq";
import { log } from "console";

Expand Down Expand Up @@ -78,45 +79,103 @@ usersRouter.post(
usersRouter.get(
"/search",
async (req: Request, res: UsersSearchErrorResponse) => {
const searchTerm = req.query.q;
const searchTerm: string = req.query.q as string;
const country: string = req.query.country as string;

if (typeof searchTerm != "string") {
if (!searchTerm && !country) {
return res.status(404).json({
status: "error",
errors: { searchTerm: "not provided" },
});
}

if (searchTerm.length == 0) {
return res
.status(404)
.json({ status: "error", errors: { searchTerm: "is empty" } });
}

try {
const page: number = parseInt((req.query.page as string) || "");
const maxUsersOnPage: number = parseInt(
(req.query.maxUsers as string) || "",
);
const session = driver.session();
const wordVec = wordToVec(searchTerm);
let allUsers: User[] = [];

if (country && searchTerm) {
const wordVec = wordToVec(searchTerm);
const searchRequest = await session.run(
`CALL db.index.vector.queryNodes('user-names', 100, $wordVec)
YIELD node AS similarUser, score
RETURN similarUser, score`,
{ wordVec },
);

allUsers = searchRequest.records
.map((r) => {
return filterUser(r.get("similarUser").properties);
})
.filter((user) => user.country === country);
} else if (country) {
const searchRequest = await session.run(
`MATCH (similarUser:User {country: $country})
RETURN similarUser`,
{ country },
);

allUsers = searchRequest.records.map((r) => {
return filterUser(r.get("similarUser").properties);
});
} else if (searchTerm) {
const wordVec = wordToVec(searchTerm);
const searchRequest = await session.run(
`CALL db.index.vector.queryNodes('user-names', 100, $wordVec)
YIELD node AS similarUser, score
RETURN similarUser, score`,
{ wordVec },
);

allUsers = searchRequest.records.map((r) => {
return filterUser(r.get("similarUser").properties);
});
}

if (wordVec.length == 0) {
return res
.status(400)
.json({ status: "error", errors: { searchTerm: "incorrect" } });
await session.close();

if (!page && !maxUsersOnPage) {
if (allUsers.length === 0) {
return res.status(404).json({
status: "error",
errors: { users: "No users found" },
});
}
const totalPage: number = roundToInt(allUsers.length / 10);
return res.status(200).json({
status: "ok",
allUsersSize: allUsers.length,
totalPage: totalPage,
users: allUsers,
});
} else if (!page || !maxUsersOnPage) {
return res.status(400).json({
status: "error",
errors: { params: "Missing or incorrect query params" },
});
}

const userRequest = await session.run(
`CALL db.index.vector.queryNodes('user-names', 10, $wordVec)
YIELD node AS similarUser, score
RETURN similarUser, score`,
{ wordVec },
const users = allUsers.slice(
(page - 1) * maxUsersOnPage,
page * maxUsersOnPage,
);
const users = userRequest.records.map((r) => {
return [
filterUser(r.get("similarUser").properties),
Number(r.get("score")),
] as [User, number];

if (users.length === 0) {
return res.status(404).json({
status: "error",
errors: { users: "No users found with given queries" },
});
}
const totalPage: number = roundToInt(allUsers.length / maxUsersOnPage);
return res.status(200).json({
status: "ok",
allUsersSize: allUsers.length,
totalPage: totalPage,
users,
});
await session.close();
return res.json({ status: "ok", users });
} catch (err) {
console.log("Error:", err);
return res.status(404).json({ status: "error", errors: err as object });
Expand All @@ -143,36 +202,6 @@ usersRouter.get("/:userId", async (req: Request, res: UserErrorResponse) => {
}
});

usersRouter.get(
"/:userId/friends",
async (req: Request, res: FriendsErrorResponse) => {
try {
const userId = req.params.userId;

const session = driver.session();
const user = await userExists(session, { id: userId });

if (!user) {
return userNotFoundRes(res);
}

const friendRequest = await session.run(
`MATCH (u:User {id: $userId})-[:IS_FRIENDS_WITH]-(f:User) RETURN f`,
{ userId },
);
await session.close();

const friends = friendRequest.records.map((f) =>
filterUser(f.get("f").properties),
);
return res.json({ status: "ok", friends });
} catch (err) {
console.log("Error:", err);
return res.status(404).json({ status: "error", errors: err as object });
}
},
);

usersRouter.get("/meetings/:userId", async (req: Request, res) => {
try {
const session = driver.session();
Expand Down
Loading
Loading