Skip to content

Commit dec91cd

Browse files
marcelklehrmarcelklehr
authored
Merge pull request #23 from nextcloud/feat/poetry-and-integration-tests
feat: use poetry to manage dependencies and add an integration test
2 parents 2c401a2 + 7b370ed commit dec91cd

File tree

7 files changed

+3008
-20
lines changed

7 files changed

+3008
-20
lines changed

‎.github/workflows/integration_test.yml‎

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
# SPDX-FileCopyrightText: Nextcloud contributors
2+
# SPDX-License-Identifier: AGPL-3.0-or-later
3+
4+
name: Integration test
5+
6+
on:
7+
pull_request:
8+
push:
9+
branches:
10+
- main
11+
- stable*
12+
13+
env:
14+
APP_NAME: context_agent
15+
16+
concurrency:
17+
group: integration-test-${{ github.head_ref || github.run_id }}
18+
cancel-in-progress: true
19+
20+
21+
jobs:
22+
transcription:
23+
runs-on: ubuntu-22.04
24+
25+
strategy:
26+
# do not stop on another job's failure
27+
fail-fast: false
28+
matrix:
29+
php-versions: [ '8.1' ]
30+
databases: [ 'sqlite' ]
31+
server-versions: [ 'master', 'stable31' ]
32+
33+
name: Integration test on ☁️${{ matrix.server-versions }} 🐘${{ matrix.php-versions }}
34+
35+
env:
36+
MYSQL_PORT: 4444
37+
PGSQL_PORT: 4445
38+
39+
PYTHONUNBUFFERED: 1
40+
APP_HOST: 0.0.0.0
41+
APP_SECRET: 12345
42+
COMPUTE_DEVICE: CPU
43+
NEXTCLOUD_URL: http://localhost:8080
44+
45+
services:
46+
mysql:
47+
image: mariadb:10.5
48+
ports:
49+
- 4444:3306/tcp
50+
env:
51+
MYSQL_ROOT_PASSWORD: rootpassword
52+
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5
53+
postgres:
54+
image: postgres
55+
ports:
56+
- 4445:5432/tcp
57+
env:
58+
POSTGRES_USER: root
59+
POSTGRES_PASSWORD: rootpassword
60+
POSTGRES_DB: nextcloud
61+
options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5
62+
63+
steps:
64+
- name: Checkout server
65+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
66+
with:
67+
repository: nextcloud/server
68+
ref: ${{ matrix.server-versions }}
69+
70+
- name: Checkout submodules
71+
shell: bash
72+
run: |
73+
auth_header="$(git config --local --get http.https://github.com/.extraheader)"
74+
git submodule sync --recursive
75+
git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1
76+
77+
- name: Set up php ${{ matrix.php-versions }}
78+
uses: shivammathur/setup-php@9e72090525849c5e82e596468b86eb55e9cc5401 # v2.32.0
79+
with:
80+
php-version: ${{ matrix.php-versions }}
81+
tools: phpunit
82+
extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip
83+
84+
- name: Checkout llm2 app
85+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
86+
with:
87+
path: llm2
88+
repository: nextcloud/llm2
89+
ref: main
90+
persist-credentials: false
91+
92+
- name: Get app version
93+
id: llm2_appinfo
94+
uses: skjnldsv/xpath-action@7e6a7c379d0e9abc8acaef43df403ab4fc4f770c # master
95+
with:
96+
filename: llm2/appinfo/info.xml
97+
expression: "/info/version/text()"
98+
99+
- name: Checkout app
100+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
101+
with:
102+
path: ${{ env.APP_NAME }}
103+
persist-credentials: false
104+
105+
- name: Get app version
106+
id: context_agent_appinfo
107+
uses: skjnldsv/xpath-action@7e6a7c379d0e9abc8acaef43df403ab4fc4f770c # master
108+
with:
109+
filename: ${{ env.APP_NAME }}/appinfo/info.xml
110+
expression: "/info/version/text()"
111+
112+
- name: Set up Nextcloud
113+
if: ${{ matrix.databases != 'pgsql'}}
114+
run: |
115+
sleep 25
116+
mkdir data
117+
./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
118+
composer run serve &
119+
120+
- name: Set up Nextcloud
121+
if: ${{ matrix.databases == 'pgsql'}}
122+
run: |
123+
sleep 25
124+
mkdir data
125+
./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
126+
composer run serve &
127+
128+
- name: Checkout app_api
129+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
130+
with:
131+
path: apps/app_api
132+
repository: nextcloud/app_api
133+
ref: ${{ matrix.server-versions == 'master' && 'main' || matrix.server-versions }}
134+
persist-credentials: false
135+
136+
- name: Enable app and app_api
137+
run: ./occ app:enable -vvv -f app_api
138+
139+
- name: Register manual deploy daemon
140+
run: |
141+
./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
142+
143+
- name: Setup python 3.10
144+
uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 #v5.5.0
145+
with:
146+
python-version: '3.10'
147+
148+
- name: Install llm2 app
149+
working-directory: llm2
150+
run: |
151+
sudo apt-get update
152+
sudo apt install pipx
153+
pipx install poetry
154+
poetry install
155+
156+
- name: Init llm2
157+
working-directory: llm2/lib
158+
env:
159+
APP_ID: llm2
160+
APP_PORT: 9080
161+
APP_VERSION: ${{ fromJson(steps.llm2_appinfo.outputs.result).version }}
162+
run: |
163+
poetry run python3 main.py > ../logs 2>&1 &
164+
165+
- name: Register backend
166+
run: |
167+
./occ app_api:app:register llm2 manual_install --json-info "{\"appid\":\"llm2\",\"name\":\"Local large language model\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.llm2_appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":9080,\"scopes\":[\"AI_PROVIDERS\", \"TASK_PROCESSING\"],\"system_app\":0}" --force-scopes --wait-finish
168+
169+
170+
- name: Install context_agent app
171+
working-directory: ${{ env.APP_NAME }}
172+
run: |
173+
sudo apt install pipx
174+
pipx install poetry
175+
poetry install
176+
177+
- name: Init context_agent
178+
working-directory: ${{ env.APP_NAME }}/ex_app/lib/
179+
env:
180+
APP_ID: ${{ env.APP_NAME }}
181+
APP_PORT: 9081
182+
APP_VERSION: ${{ fromJson(steps.context_agent_appinfo.outputs.result).version }}
183+
run: |
184+
PYTHONPATH="$(pwd)/../.." poetry run python3 main.py > ../../logs 2>&1 &
185+
186+
- name: Register backend
187+
run: |
188+
./occ app_api:app:register context_agent manual_install --json-info "{\"appid\":\"context_agent\",\"name\":\"Nextcloud Assistant Context Agent\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.context_agent_appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":9081,\"scopes\":[\"AI_PROVIDERS\", \"TASK_PROCESSING\"],\"system_app\":0}" --force-scopes --wait-finish
189+
190+
191+
- name: Run task
192+
env:
193+
CREDS: "admin:password"
194+
run: |
195+
sleep 300
196+
TASK=$(curl -X POST -u "$CREDS" -H "oCS-APIRequest: true" -H "Content-type: application/json" http://localhost:8080/ocs/v2.php/taskprocessing/schedule?format=json --data-raw '{"input": {"input": "What are the coordinates of Berlin, Germany?", "confirmation":1, "conversation_token": ""},"type":"core:contextagent:interaction", "appId": "test", "customId": ""}')
197+
echo $TASK
198+
TASK_ID=$(echo $TASK | jq '.ocs.data.task.id')
199+
NEXT_WAIT_TIME=0
200+
TASK_STATUS='"STATUS_SCHEDULED"'
201+
until [ $NEXT_WAIT_TIME -eq 59 ] || [ "$TASK_STATUS" == '"STATUS_SUCCESSFUL"' ] || [ "$TASK_STATUS" == '"STATUS_FAILED"' ]; do
202+
TASK=$(curl -u "$CREDS" -H "oCS-APIRequest: true" http://localhost:8080/ocs/v2.php/taskprocessing/task/$TASK_ID?format=json)
203+
echo $TASK
204+
TASK_STATUS=$(echo $TASK | jq '.ocs.data.task.status')
205+
echo $TASK_STATUS
206+
echo "Sleeping for $NEXT_WAIT_TIME seconds"
207+
sleep $(( NEXT_WAIT_TIME++ ))
208+
done
209+
curl -u "$CREDS" -H "oCS-APIRequest: true" http://localhost:8080/ocs/v2.php/taskprocessing/task/$TASK_ID?format=json
210+
[ "$TASK_STATUS" == '"STATUS_SUCCESSFUL"' ]
211+
echo $TASK | jq '.ocs.data.task.output.output'
212+
echo $TASK | jq '.ocs.data.task.output.sources'
213+
echo $TASK | jq '.ocs.data.task.output.sources' | grep -q get_coordinates_for_address
214+
echo $TASK | jq '.ocs.data.task.output.output' | grep -q '52.'
215+
echo $TASK | jq '.ocs.data.task.output.output' | grep -q '13.'
216+
217+
- name: Show nextcloud logs
218+
if: always()
219+
run: |
220+
tail data/nextcloud.log
221+
222+
- name: Show context_chat logs
223+
if: always()
224+
run: |
225+
[ -f ${{ env.APP_NAME }}/logs ] && cat ${{ env.APP_NAME }}/logs || echo "No context_chat logs"
226+
227+
- name: Show llm2 logs
228+
if: always()
229+
run: |
230+
[ -f llm2/logs ] && cat llm2/logs || echo "No llm2 logs"

‎Dockerfile‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,15 @@ RUN set -ex; \
2525
chmod +x /usr/local/bin/frpc; \
2626
rm -rf /tmp/frp /tmp/frp.tar.gz
2727

28-
COPY requirements.txt /
2928

30-
RUN \
31-
python3 -m pip install -r requirements.txt && rm -rf ~/.cache && rm requirements.txt
29+
RUN apt install -y pipx build-essential git vim
30+
RUN pipx install poetry
31+
32+
# Install requirements
33+
34+
COPY pyproject.toml .
35+
COPY poetry.lock .
36+
RUN poetry install
3237

3338
ADD /ex_app/cs[s] /ex_app/css
3439
ADD /ex_app/im[g] /ex_app/img
@@ -41,5 +46,5 @@ COPY --chmod=775 start.sh /
4146

4247
WORKDIR /ex_app/lib
4348
ENV PYTHONPATH="/"
44-
ENTRYPOINT ["/start.sh", "python3", "main.py"]
49+
ENTRYPOINT ["/start.sh", "poetry", "run", "python3", "main.py"]
4550
HEALTHCHECK --interval=2s --timeout=2s --retries=300 CMD /healthcheck.sh

‎REUSE.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,9 @@ path = ".github/renovate.json"
1010
precedence = "aggregate"
1111
SPDX-FileCopyrightText = "2024 Nextcloud GmbH and Nextcloud contributors"
1212
SPDX-License-Identifier = "AGPL-3.0-or-later"
13+
14+
[[annotations]]
15+
path = "poetry.lock"
16+
precedence = "aggregate"
17+
SPDX-FileCopyrightText = "2024 Nextcloud GmbH and Nextcloud contributors"
18+
SPDX-License-Identifier = "AGPL-3.0-or-later"

‎ex_app/lib/nc_model.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ class ChatWithNextcloud(BaseChatModel):
3636
nc: Nextcloud = NextcloudApp()
3737
tools: Sequence[
3838
Union[typing.Dict[str, Any], type, Callable, BaseTool]] = []
39+
TIMEOUT: int = 60 * 20 # 20 minutes
3940

4041
def _generate(
4142
self,
@@ -96,9 +97,10 @@ def _generate(
9697
log(nc, LogLvl.DEBUG, task)
9798

9899
i = 0
99-
# wait for 5 seconds * 60 * 2 = 10 minutes (one i ^= 5 sec)
100-
while task.status != "STATUS_SUCCESSFUL" and task.status != "STATUS_FAILED" and i < 60 * 2:
101-
time.sleep(5)
100+
wait_time = 5
101+
# wait for TIMEOUT (one i ^= 5 sec)
102+
while task.status != "STATUS_SUCCESSFUL" and task.status != "STATUS_FAILED" and i < self.TIMEOUT / wait_time:
103+
time.sleep(wait_time)
102104
i += 1
103105
try:
104106
response = nc.ocs("GET", f"/ocs/v1.php/taskprocessing/task/{task.id}")
@@ -124,9 +126,12 @@ def _generate(
124126
except ValidationError as e:
125127
raise Exception("Failed to parse Nextcloud TaskProcessing task result") from e
126128

127-
if task.status != "STATUS_SUCCESSFUL":
129+
if task.status == "STATUS_FAILED":
128130
raise Exception("Nextcloud TaskProcessing Task failed")
129131

132+
if task.status in ("STATUS_RUNNING", "STATUS_SCHEDULED"):
133+
raise Exception("Nextcloud TaskProcessing Task timed out")
134+
130135
if not isinstance(task.output, dict) or "output" not in task.output:
131136
raise Exception('"output" key not found in Nextcloud TaskProcessing task result')
132137

0 commit comments

Comments
 (0)