Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: allow rows parameter and no pagination (get first n elements) #73

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+21 −3
Open

feat: allow rows parameter and no pagination (get first n elements) #73

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a47e4ab
feat: allow rows parameter and no pagination (get first n elements)
manuelguzmandao Dec 10, 2024
fdc0828
fix: add pagination control when paginating with offset (CURSOR_AS_IT…
manuelguzmandao Dec 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions crossref/restful.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ def __init__( # noqa: PLR0913
throttle=True,
crossref_plus_token=None,
timeout=30,
paginate=True,
):
self.do_http_request = HTTPRequest(throttle=throttle).do_http_request
self.etiquette = etiquette or Etiquette()
Expand All @@ -136,6 +137,7 @@ def __init__( # noqa: PLR0913
self.request_params = request_params or {}
self.context = context or ""
self.timeout = timeout
self.paginate = paginate

@property
def _rate_limits(self):
Expand Down Expand Up @@ -305,7 +307,8 @@ def __iter__(self): # noqa: PLR0912 - To many branches is not a problem.
if self.CURSOR_AS_ITER_METHOD is True:
request_params = dict(self.request_params)
request_params["cursor"] = "*"
request_params["rows"] = LIMIT
if "rows" not in request_params:
request_params["rows"] = LIMIT
while True:
result = self.do_http_request(
"get",
Expand All @@ -326,11 +329,15 @@ def __iter__(self): # noqa: PLR0912 - To many branches is not a problem.
for item in result["message"]["items"]:
yield item

if not self.paginate:
return

request_params["cursor"] = result["message"]["next-cursor"]
else:
request_params = dict(self.request_params)
request_params["offset"] = 0
request_params["rows"] = LIMIT
real_limit = request_params["rows"] if "rows" in request_params else LIMIT
request_params["rows"] = real_limit
while True:
result = self.do_http_request(
"get",
Expand All @@ -351,7 +358,10 @@ def __iter__(self): # noqa: PLR0912 - To many branches is not a problem.
for item in result["message"]["items"]:
yield item

request_params["offset"] += LIMIT
if not self.paginate:
return

request_params["offset"] += real_limit

if request_params["offset"] >= MAXOFFSET:
msg = "Offset exceded the max offset of %d"
Expand Down Expand Up @@ -621,6 +631,7 @@ def order(self, order="asc"):
context=context,
etiquette=self.etiquette,
timeout=self.timeout,
paginate=self.paginate,
)

def select(self, *args):
Expand Down Expand Up @@ -725,6 +736,7 @@ def select(self, *args):
context=context,
etiquette=self.etiquette,
timeout=self.timeout,
paginate=self.paginate,
)

def sort(self, sort="score"):
Expand Down Expand Up @@ -786,6 +798,7 @@ def sort(self, sort="score"):
context=context,
etiquette=self.etiquette,
timeout=self.timeout,
paginate=self.paginate,
)

def filter(self, **kwargs): # noqa: A003
Expand Down Expand Up @@ -843,6 +856,7 @@ def filter(self, **kwargs): # noqa: A003
context=context,
etiquette=self.etiquette,
timeout=self.timeout,
paginate=self.paginate,
)

def facet(self, facet_name, facet_count=100):
Expand Down Expand Up @@ -1161,6 +1175,7 @@ def query(self, *args):
request_params=request_params,
etiquette=self.etiquette,
timeout=self.timeout,
paginate=self.paginate,
)

def filter(self, **kwargs): # noqa: A003
Expand Down Expand Up @@ -1217,6 +1232,7 @@ def filter(self, **kwargs): # noqa: A003
context=context,
etiquette=self.etiquette,
timeout=self.timeout,
paginate=self.paginate,
)

def funder(self, funder_id, only_message=True):
Expand Down Expand Up @@ -1375,6 +1391,7 @@ def query(self, *args):
context=context,
etiquette=self.etiquette,
timeout=self.timeout,
paginate=self.paginate,
)

def filter(self, **kwargs): # noqa: A003
Expand Down Expand Up @@ -1435,6 +1452,7 @@ def filter(self, **kwargs): # noqa: A003
context=context,
etiquette=self.etiquette,
timeout=self.timeout,
paginate=self.paginate,
)

def member(self, member_id, only_message=True):
Expand Down