Add pagination support to the API (#211)

## Ticket

Resolves
#210

## Changes

This adds pagination support to the API schema and DB queries

Added a new `POST /users/search` endpoint to have an example of a
paginated endpoint.

## Context for reviewers

There are likely more features that could be built ontop of this
(multi-field sorting, Paginator class as an iterator, and a few other
utilities), but was focused on getting the core functionality of
pagination working in a fairly general manner.

This approach for pagination is based on a mix of past projects and
partially based on the
[Flask-SQLAlchemy](https://github.com/pallets-eco/flask-sqlalchemy/blob/d349bdb6229fb5893ddfc7a6ff273425e4c1da7a/src/flask_sqlalchemy/pagination.py)
libraries approach.

## Testing

Added a bunch of users locally by calling the POST /users endpoint, but
only one which would be found by the following query:
```json
{
  "is_active": true,
  "paging": {
    "page_offset": 1,
    "page_size": 25
  },
  "phone_number": "123-456-7890",
  "role_type": "USER",
  "sorting": {
    "order_by": "id",
    "sort_direction": "ascending"
  }
}
```
And got the following response (with the data removed as it's a lot):
```json
{
  "data": [...],
  "errors": [],
  "message": "Success",
  "pagination_info": {
    "order_by": "id",
    "page_offset": 1,
    "page_size": 25,
    "sort_direction": "ascending",
    "total_pages": 2,
    "total_records": 41
  },
  "status_code": 200,
  "warnings": []
}
```

Further testing was done, and can be seen in the unit tests to verify
the paging/sorting behavior.

---------

Co-authored-by: nava-platform-bot <[email protected]>

Author: chouinar

app/openapi.generated.yml

@@ -36,6 +36,10 @@ paths:
                     type: array
                     items:
                       $ref: '#/components/schemas/ValidationError'
+                  pagination_info:
+                    description: The pagination information for paginated endpoints
+                    allOf:
+                    - $ref: '#/components/schemas/PaginationInfo'
           description: Successful response
         '503':
           content:
@@ -72,6 +76,10 @@ paths:
                     type: array
                     items:
                       $ref: '#/components/schemas/ValidationError'
+                  pagination_info:
+                    description: The pagination information for paginated endpoints
+                    allOf:
+                    - $ref: '#/components/schemas/PaginationInfo'
           description: Successful response
         '422':
           content:
@@ -95,6 +103,61 @@ paths:
               $ref: '#/components/schemas/User'
       security:
       - ApiKeyAuth: []
+  /v1/users/search:
+    post:
+      parameters: []
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  message:
+                    type: string
+                    description: The message to return
+                  data:
+                    type: array
+                    items:
+                      $ref: '#/components/schemas/User'
+                  status_code:
+                    type: integer
+                    description: The HTTP status code
+                  warnings:
+                    type: array
+                    items:
+                      $ref: '#/components/schemas/ValidationError'
+                  errors:
+                    type: array
+                    items:
+                      $ref: '#/components/schemas/ValidationError'
+                  pagination_info:
+                    description: The pagination information for paginated endpoints
+                    allOf:
+                    - $ref: '#/components/schemas/PaginationInfo'
+          description: Successful response
+        '422':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ValidationError'
+          description: Validation error
+        '401':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPError'
+          description: Authentication error
+      tags:
+      - User
+      summary: User Search
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/UserSearch'
+      security:
+      - ApiKeyAuth: []
   /v1/users/{user_id}:
     get:
       parameters:
@@ -126,6 +189,10 @@ paths:
                     type: array
                     items:
                       $ref: '#/components/schemas/ValidationError'
+                  pagination_info:
+                    description: The pagination information for paginated endpoints
+                    allOf:
+                    - $ref: '#/components/schemas/PaginationInfo'
           description: Successful response
         '401':
           content:
@@ -174,6 +241,10 @@ paths:
                     type: array
                     items:
                       $ref: '#/components/schemas/ValidationError'
+                  pagination_info:
+                    description: The pagination information for paginated endpoints
+                    allOf:
+                    - $ref: '#/components/schemas/PaginationInfo'
           description: Successful response
         '422':
           content:
@@ -224,6 +295,34 @@ components:
         value:
           type: string
           description: The value that failed
+    PaginationInfo:
+      type: object
+      properties:
+        page_offset:
+          type: integer
+          description: The page number that was fetched
+          example: 1
+        page_size:
+          type: integer
+          description: The size of the page fetched
+          example: 25
+        total_records:
+          type: integer
+          description: The total number of records fetchable
+          example: 42
+        total_pages:
+          type: integer
+          description: The total number of pages that can be fetched
+          example: 2
+        order_by:
+          type: string
+          description: The field that the records were sorted by
+          example: id
+        sort_direction:
+          description: The direction the records are sorted
+          enum:
+          - ascending
+          - descending
     HTTPError:
       properties:
         detail:
@@ -260,9 +359,9 @@ components:
           description: The user's last name
         phone_number:
           type: string
+          pattern: ^([0-9]|\*){3}\-([0-9]|\*){3}\-[0-9]{4}$
           description: The user's phone number
           example: 123-456-7890
-          pattern: ^([0-9]|\*){3}\-([0-9]|\*){3}\-[0-9]{4}$
         date_of_birth:
           type: string
           format: date
@@ -289,6 +388,60 @@ components:
       - last_name
       - phone_number
       - roles
+    UserSorting:
+      type: object
+      properties:
+        order_by:
+          type: string
+          enum:
+          - id
+          - created_at
+          - updated_at
+          description: The field to sort the response by
+        sort_direction:
+          description: Whether to sort the response ascending or descending
+          enum:
+          - ascending
+          - descending
+      required:
+      - order_by
+      - sort_direction
+    Pagination:
+      type: object
+      properties:
+        page_size:
+          type: integer
+          minimum: 1
+          description: The size of the page to fetch
+          example: 25
+        page_offset:
+          type: integer
+          minimum: 1
+          description: The page number to fetch, starts counting from 1
+          example: 1
+      required:
+      - page_offset
+      - page_size
+    UserSearch:
+      type: object
+      properties:
+        phone_number:
+          type: string
+          pattern: ^([0-9]|\*){3}\-([0-9]|\*){3}\-[0-9]{4}$
+          description: The user's phone number
+          example: 123-456-7890
+        is_active:
+          type: boolean
+        role_type:
+          enum:
+          - USER
+          - ADMIN
+        sorting:
+          $ref: '#/components/schemas/UserSorting'
+        paging:
+          $ref: '#/components/schemas/Pagination'
+      required:
+      - paging
     UserUpdate:
       type: object
       properties:
@@ -307,9 +460,9 @@ components:
           description: The user's last name
         phone_number:
           type: string
+          pattern: ^([0-9]|\*){3}\-([0-9]|\*){3}\-[0-9]{4}$
           description: The user's phone number
           example: 123-456-7890
-          pattern: ^([0-9]|\*){3}\-([0-9]|\*){3}\-[0-9]{4}$
         date_of_birth:
           type: string
           format: date

app/src/api/healthcheck.py

@@ -23,11 +23,11 @@ class HealthcheckSchema(request_schema.OrderedSchema):
 @healthcheck_blueprint.get("/health")
 @healthcheck_blueprint.output(HealthcheckSchema)
 @healthcheck_blueprint.doc(responses=[200, ServiceUnavailable.code])
-def health() -> Tuple[dict, int]:
+def health() -> Tuple[response.ApiResponse, int]:
     try:
         with flask_db.get_db(current_app).get_connection() as conn:
             assert conn.scalar(text("SELECT 1 AS healthy")) == 1
-        return response.ApiResponse(message="Service healthy").asdict(), 200
+        return response.ApiResponse(message="Service healthy"), 200
     except Exception:
         logger.exception("Connection to DB failure")
-        return response.ApiResponse(message="Service unavailable").asdict(), ServiceUnavailable.code
+        return response.ApiResponse(message="Service unavailable"), ServiceUnavailable.code

app/src/api/response.py

@@ -1,8 +1,7 @@
 import dataclasses
-from typing import Optional
+from typing import Any, Optional
 
-from src.api.schemas import response_schema
-from src.db.models.base import Base
+from src.pagination.pagination_models import PaginationInfo
 
 
 @dataclasses.dataclass
@@ -33,16 +32,9 @@ class ApiResponse:
     """Base response model for all API responses."""
 
     message: str
-    data: Optional[Base] = None
+    data: Optional[Any] = None
     warnings: list[ValidationErrorDetail] = dataclasses.field(default_factory=list)
     errors: list[ValidationErrorDetail] = dataclasses.field(default_factory=list)
+    status_code: int = 200
 
-    # This method is used to convert ApiResponse objects to a dictionary
-    # This is necessary because APIFlask has a bug that causes an exception to be
-    # thrown when returning objects from routes when BASE_RESPONSE_SCHEMA is set
-    # (See https://github.com/apiflask/apiflask/issues/384)
-    # Once that issue is fixed, this method can be removed and routes can simply
-    # return ApiResponse objects directly and allow APIFlask to serealize the objects
-    # to JSON automatically.
-    def asdict(self) -> dict:
-        return response_schema.ResponseSchema().dump(self)
+    pagination_info: PaginationInfo | None = None

app/src/api/schemas/response_schema.py

@@ -1,6 +1,7 @@
 from apiflask import fields
 
 from src.api.schemas import request_schema
+from src.pagination.pagination_schema import PaginationInfoSchema
 
 
 class ValidationErrorSchema(request_schema.OrderedSchema):
@@ -15,5 +16,10 @@ class ResponseSchema(request_schema.OrderedSchema):
     message = fields.String(metadata={"description": "The message to return"})
     data = fields.Field(metadata={"description": "The REST resource object"}, dump_default={})
     status_code = fields.Integer(metadata={"description": "The HTTP status code"}, dump_default=200)
-    warnings = fields.List(fields.Nested(ValidationErrorSchema), dump_default=[])
-    errors = fields.List(fields.Nested(ValidationErrorSchema), dump_default=[])
+    warnings = fields.List(fields.Nested(ValidationErrorSchema()), dump_default=[])
+    errors = fields.List(fields.Nested(ValidationErrorSchema()), dump_default=[])
+
+    pagination_info = fields.Nested(
+        PaginationInfoSchema(),
+        metadata={"description": "The pagination information for paginated endpoints"},
+    )

app/src/api/users/user_routes.py

@@ -19,13 +19,13 @@
 @user_blueprint.output(user_schemas.UserSchema, status_code=201)
 @user_blueprint.auth_required(api_key_auth)
 @flask_db.with_db_session()
-def user_post(db_session: db.Session, user_params: users.CreateUserParams) -> dict:
+def user_post(db_session: db.Session, user_params: users.CreateUserParams) -> response.ApiResponse:
     """
     POST /v1/users
     """
     user = user_service.create_user(db_session, user_params)
     logger.info("Successfully inserted user", extra=get_user_log_params(user))
-    return response.ApiResponse(message="Success", data=user).asdict()
+    return response.ApiResponse(message="Success", data=user)
 
 
 @user_blueprint.patch("/v1/users/<uuid:user_id>")
@@ -38,20 +38,34 @@ def user_post(db_session: db.Session, user_params: users.CreateUserParams) -> di
 @flask_db.with_db_session()
 def user_patch(
     db_session: db.Session, user_id: str, patch_user_params: users.PatchUserParams
-) -> dict:
+) -> response.ApiResponse:
     user = user_service.patch_user(db_session, user_id, patch_user_params)
     logger.info("Successfully patched user", extra=get_user_log_params(user))
-    return response.ApiResponse(message="Success", data=user).asdict()
+    return response.ApiResponse(message="Success", data=user)
 
 
 @user_blueprint.get("/v1/users/<uuid:user_id>")
 @user_blueprint.output(user_schemas.UserSchema)
 @user_blueprint.auth_required(api_key_auth)
 @flask_db.with_db_session()
-def user_get(db_session: db.Session, user_id: str) -> dict:
+def user_get(db_session: db.Session, user_id: str) -> response.ApiResponse:
     user = user_service.get_user(db_session, user_id)
     logger.info("Successfully fetched user", extra=get_user_log_params(user))
-    return response.ApiResponse(message="Success", data=user).asdict()
+    return response.ApiResponse(message="Success", data=user)
+
+
+@user_blueprint.post("/v1/users/search")
+@user_blueprint.input(user_schemas.UserSearchSchema, arg_name="search_params")
+# many=True allows us to return a list of user objects
+@user_blueprint.output(user_schemas.UserSchema(many=True))
+@user_blueprint.auth_required(api_key_auth)
+@flask_db.with_db_session()
+def user_search(db_session: db.Session, search_params: dict) -> response.ApiResponse:
+    user_result, pagination_info = user_service.search_user(db_session, search_params)
+    logger.info("Successfully searched users")
+    return response.ApiResponse(
+        message="Success", data=user_result, pagination_info=pagination_info
+    )
 
 
 def get_user_log_params(user: User) -> dict[str, Any]:

app/src/api/users/user_schemas.py

@@ -1,8 +1,11 @@
-from apiflask import fields
+from apiflask import fields, validators
 from marshmallow import fields as marshmallow_fields
 
 from src.api.schemas import request_schema
 from src.db.models import user_models
+from src.pagination.pagination_schema import PaginationSchema, generate_sorting_schema
+
+PHONE_NUMBER_VALIDATOR = validators.Regexp(r"^([0-9]|\*){3}\-([0-9]|\*){3}\-[0-9]{4}$")
 
 
 class RoleSchema(request_schema.OrderedSchema):
@@ -23,10 +26,10 @@ class UserSchema(request_schema.OrderedSchema):
     last_name = fields.String(metadata={"description": "The user's last name"}, required=True)
     phone_number = fields.String(
         required=True,
+        validate=[PHONE_NUMBER_VALIDATOR],
         metadata={
             "description": "The user's phone number",
             "example": "123-456-7890",
-            "pattern": r"^([0-9]|\*){3}\-([0-9]|\*){3}\-[0-9]{4}$",
         },
     )
     date_of_birth = fields.Date(
@@ -37,8 +40,26 @@ class UserSchema(request_schema.OrderedSchema):
         metadata={"description": "Whether the user is active"},
         required=True,
     )
-    roles = fields.List(fields.Nested(RoleSchema), required=True)
+    roles = fields.List(fields.Nested(RoleSchema()), required=True)
 
     # Output only fields in addition to id field
     created_at = fields.DateTime(dump_only=True)
     updated_at = fields.DateTime(dump_only=True)
+
+
+class UserSearchSchema(request_schema.OrderedSchema):
+    # Fields that you can search for users by, only includes a subset of user fields
+    phone_number = fields.String(
+        validate=[PHONE_NUMBER_VALIDATOR],
+        metadata={
+            "description": "The user's phone number",
+            "example": "123-456-7890",
+        },
+    )
+
+    is_active = fields.Boolean()
+
+    role_type = fields.Enum(user_models.RoleType, by_value=True)
+
+    sorting = fields.Nested(generate_sorting_schema("UserSortingSchema")())
+    paging = fields.Nested(PaginationSchema(), required=True)