Skip to content

Commit

Permalink
fix: attempt to fix saving task hanging issue
Browse files Browse the repository at this point in the history
  • Loading branch information
tikazyq committed Dec 21, 2022
1 parent b42ee97 commit c0adb83
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 12 deletions.
12 changes: 1 addition & 11 deletions _ext/python/crawlab/result.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import json
from queue import Queue
from typing import List, Optional, Dict

from crawlab.client import get_client, Client
Expand All @@ -15,12 +14,10 @@ class ResultService:
# internal
c: Client = None
task_stub: TaskServiceStub = None
queue: Queue = Queue()

def __init__(self):
self.c = get_client()
self.task_stub = self.c.task_service_stub
self.task_stub.Subscribe.future(self.yield_msg())

def save_item(self, *items: Dict):
self.save(list(items))
Expand Down Expand Up @@ -59,14 +56,7 @@ def _save(self, items: List[Dict]):
code=INSERT_DATA,
data=data,
)
self.enqueue(msg)

def enqueue(self, msg: StreamMessage):
self.queue.put(msg)

def yield_msg(self):
while True:
yield self.queue.get()
self.task_stub.Subscribe(iter([msg]))


RS: Optional[ResultService] = None
Expand Down
2 changes: 1 addition & 1 deletion _ext/python/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

setup(
name='crawlab-sdk',
version='0.6.0-8',
version='0.6.0-9',
packages=find_packages(),
url='https://github.com/crawlab-team/crawlab-sdk',
license='BSD-3-Clause',
Expand Down

0 comments on commit c0adb83

Please sign in to comment.