@@ -46,7 +46,7 @@ def trim_string(
46
46
return trimmed_string
47
47
48
48
49
- def prep_docs (input_file : str , max_docs :int = max_docs ):
49
+ def prep_docs (input_file : str , num_docs :int = max_docs ):
50
50
"""
51
51
Create generator for every row in csv as a Document
52
52
:param input_file: Input csv filename
@@ -56,7 +56,7 @@ def prep_docs(input_file: str, max_docs:int=max_docs):
56
56
with open (input_file , "r" ) as csv_file :
57
57
csv_reader = csv .DictReader (csv_file )
58
58
input_field = "Description"
59
- for row in itertools .islice (csv_reader , max_docs ):
59
+ for row in itertools .islice (csv_reader , num_docs ):
60
60
# Fix invalid ratings and counts
61
61
if row ["Average User Rating" ] == "" :
62
62
row ["Average User Rating" ] = random .uniform (0.0 , 5.0 )
@@ -70,7 +70,7 @@ def prep_docs(input_file: str, max_docs:int=max_docs):
70
70
yield doc
71
71
72
72
73
- def index ():
73
+ def index (num_docs = max_docs ):
74
74
flow = (
75
75
Flow ()
76
76
# .add(uses='jinahub+docker://TransformerTorchEncoder', pretrained_model_name_or_path="sentence-transformers/msmarco-distilbert-base-v3", name="encoder", max_length=50)
@@ -84,7 +84,7 @@ def index():
84
84
with flow :
85
85
flow .post (
86
86
on = "/index" ,
87
- inputs = prep_docs (input_file = backend_datafile , max_docs = max_docs ),
87
+ inputs = prep_docs (input_file = backend_datafile , num_docs = num_docs ),
88
88
request_size = 64 ,
89
89
read_mode = "r" ,
90
90
)
@@ -131,7 +131,7 @@ def main(task: str, num_docs: int, force: bool):
131
131
\n +----------------------------------------------------------------------------------+"
132
132
)
133
133
sys .exit (1 )
134
- index ()
134
+ index (num_docs = num_docs )
135
135
if task == "query_restful" :
136
136
if not os .path .exists (workspace ):
137
137
print (
0 commit comments