Skip to content

Commit 5489cb7

Browse files
nicoloboschinicoloboschi
authored
refactor: split repository in subpackages (#376)
1 parent f1ce23e commit 5489cb7

120 files changed

Lines changed: 828 additions & 212 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/add-report-to-github-summary/action.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@ runs:
2929
fi
3030
}
3131
if [ "${{ inputs.all }}" == "true" ]; then
32-
write_report ragstack-e2e-tests/all-tests-report.txt "All tests"
32+
write_report libs/e2e-tests/all-tests-report.txt "All tests"
3333
fi
3434
if [ "${{ inputs.langchain }}" == "true" ]; then
35-
write_report ragstack-e2e-tests/langchain-tests-report.txt "LangChain tests"
35+
write_report libs/e2e-tests/langchain-tests-report.txt "LangChain tests"
3636
fi
3737
if [ "${{ inputs.llama-index }}" == "true" ]; then
38-
write_report ragstack-e2e-tests/llamaindex-tests-report.txt "Llama Index tests"
38+
write_report libs/e2e-tests/llamaindex-tests-report.txt "Llama Index tests"
3939
fi

‎.github/actions/lint/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ runs:
1313
- name: Lint
1414
shell: bash
1515
run: |
16-
tox -c ragstack-e2e-tests -e lint
16+
tox -c libs/e2e-tests -e lint
1717
tox -e lint-yaml
1818
1919
- name: Notebook

‎.github/changes-filter.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
# https://github.com/dorny/paths-filter
22
e2e_tests:
3-
- added|deleted|modified: "ragstack-e2e-tests/**"
3+
- "libs/e2e-tests/**"
44
- "pyproject.toml"
5+
- "**/pyproject.toml"
56
- "./.github/actions/**"
67
- "./.github/workflows/ci-e2e-tests.yml"
78
- "./.github/workflows/_run_e2e_tests.yml"

‎.github/workflows/_run_e2e_tests.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -125,13 +125,13 @@ jobs:
125125
# workaround for FileLocked https://github.com/datastax/ragstack-ai/actions/runs/8629355336/job/23653432063?pr=316
126126
rm -rf **/.tox
127127
if [ "${{ inputs.suite-name == 'ragstack' }}" == "true" ]; then
128-
tox -c ragstack-e2e-tests
128+
tox -c libs/e2e-tests
129129
elif [ "${{ inputs.suite-name == 'langchain' }}" == "true" ]; then
130-
tox -c ragstack-e2e-tests -e langchain
130+
tox -c libs/e2e-tests -e langchain
131131
elif [ "${{ inputs.suite-name == 'llamaindex' }}" == "true" ]; then
132-
tox -c ragstack-e2e-tests -e llamaindex
132+
tox -c libs/e2e-tests -e llamaindex
133133
elif [ "${{ inputs.suite-name == 'ragstack-latest-release' }}" == "true" ]; then
134-
tox -c ragstack-e2e-tests -e ragstack-ai-latest
134+
tox -c libs/e2e-tests -e ragstack-ai-latest
135135
else
136136
echo "Unknown suite name: ${{ inputs.suite-name }}"
137137
exit 1
@@ -150,15 +150,15 @@ jobs:
150150
echo "commit-url=https://github.com/datastax/ragstack-ai/commits/${commit_ref}" >> $GITHUB_OUTPUT
151151
152152
elif [ "${{ inputs.suite-name == 'langchain' }}" == "true" ]; then
153-
commit_ref=$(grep resolved_reference ragstack-e2e-tests/poetry.lock | awk -F'"' '{print $2}' | head -1)
153+
commit_ref=$(grep resolved_reference libs/e2e-tests/poetry.lock | awk -F'"' '{print $2}' | head -1)
154154
echo "commit-url=https://github.com/langchain-ai/langchain/commits/${commit_ref}" >> $GITHUB_OUTPUT
155155
156156
elif [ "${{ inputs.suite-name == 'llamaindex' }}" == "true" ]; then
157-
commit_ref=$(grep resolved_reference ragstack-e2e-tests/poetry.lock | awk -F'"' '{print $2}' | head -1)
157+
commit_ref=$(grep resolved_reference libs/e2e-tests/poetry.lock | awk -F'"' '{print $2}' | head -1)
158158
echo "commit-url=https://github.com/run-llama/llama_index/commits/${commit_ref}" >> $GITHUB_OUTPUT
159159
160160
elif [ "${{ inputs.suite-name == 'ragstack-latest-release' }}" == "true" ]; then
161-
(tox run -c ragstack-e2e-tests -e get-latest-ragstack-ai-version -q) | tail -3 | head -n 1 > /tmp/ragstack-ai-version
161+
(tox run -c libs/e2e-tests -e get-latest-ragstack-ai-version -q) | tail -3 | head -n 1 > /tmp/ragstack-ai-version
162162
cat /tmp/ragstack-ai-version
163163
ragstack_version=$(cat /tmp/ragstack-ai-version)
164164
echo "commit-url=https://pypi.org/project/ragstack-ai/${ragstack_version}" >> $GITHUB_OUTPUT
@@ -168,7 +168,7 @@ jobs:
168168
if: ${{ always() && inputs.deploy-to-slack }}
169169
uses: ./.github/actions/deploy-slack-report
170170
with:
171-
from-report-file: "ragstack-e2e-tests/failed-tests-report.txt"
171+
from-report-file: "libs/e2e-tests/failed-tests-report.txt"
172172
type: "${{ inputs.title }}"
173173
outcome: ${{ steps.e2e-tests.outcome }}
174174
commit-url: "${{ steps.info.outputs.commit-url }}"
@@ -180,7 +180,7 @@ jobs:
180180
with:
181181
report-type: tests
182182
token: ${{ secrets.TESTSPACE_TOKEN }}
183-
report-file: ragstack-e2e-tests/results.xml
183+
report-file: libs/e2e-tests/results.xml
184184
space: ${{ inputs.testspace-space }}
185185

186186
- name: Cleanup AstraDB

‎.github/workflows/ci-security-scans.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@ jobs:
4545
- name: Prepare dev docker image
4646
if: ${{ always() }}
4747
run: |
48-
poetry build
49-
docker build -t datastax/ragstack-ai:latest-dev -f docker/Dockerfile.dev dist
48+
docker build -t datastax/ragstack-ai:latest-dev -f docker/Dockerfile.dev .
5049
5150
- name: Run Snyk to check Docker image for vulnerabilities
5251
uses: snyk/actions/docker@master
@@ -114,7 +113,7 @@ jobs:
114113
mv snyk.json snyk_reports/snyk-docker.json
115114
116115
- name: "Setup: Python 3.11"
117-
if: ${{ always() }}
116+
if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
118117
uses: ./.github/actions/setup-python
119118

120119
- name: Testspace deploy report

‎.github/workflows/ci-unit-tests.yml‎

Lines changed: 51 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -44,17 +44,22 @@ jobs:
4444
- name: Run lint
4545
uses: ./.github/actions/lint
4646

47-
- name: "Build"
48-
run: |
49-
tox -e build
47+
- name: "Unit tests (root)"
48+
run: tox -e unit-tests
5049

51-
- name: "Unit tests"
52-
run: |
53-
tox -e unit-tests
50+
- name: "Unit tests (colbert)"
51+
run: tox -e unit-tests -c libs/colbert
52+
53+
- name: "Unit tests (langchain)"
54+
run: tox -e unit-tests -c libs/langchain
55+
56+
- name: "Unit tests (llamaindex)"
57+
run: tox -e unit-tests -c libs/llamaindex
5458

5559
integration-tests:
5660
name: Integration Tests
5761
runs-on: ubuntu-latest
62+
timeout-minutes: 30
5863
steps:
5964
- name: Check out the repo
6065
uses: actions/checkout@v4
@@ -71,7 +76,6 @@ jobs:
7176
- name: Setup AstraDB
7277
uses: ./.github/actions/setup-astra-db
7378
id: astra-db
74-
timeout-minutes: 10
7579
with:
7680
astra-token: ${{ secrets.E2E_TESTS_ASTRA_DEV_DB_TOKEN }}
7781
db-name: ${{ steps.db-name.outputs.db-name }}
@@ -80,14 +84,51 @@ jobs:
8084
cloud: "aws"
8185

8286
- name: "Integration tests"
83-
timeout-minutes: 10
8487
env:
8588
ASTRA_DB_TOKEN: "${{ secrets.E2E_TESTS_ASTRA_DEV_DB_TOKEN }}"
8689
ASTRA_DB_ID: "${{ steps.astra-db.outputs.db_id }}"
8790
ASTRA_DB_ENV: "DEV"
8891
run: |
89-
rm -rf .tox
90-
tox -e integration-tests
92+
set -ex
93+
du -sh ~/*
94+
du -sh *
95+
du -sh libs/*
96+
du -sh libs/colbert/*
97+
du -sh libs/langchain/*
98+
du -sh libs/llamaindex/*
99+
rm -rf **/.tox
100+
du -sh ~/*
101+
du -sh *
102+
du -sh libs/*
103+
du -sh libs/colbert/*
104+
du -sh libs/langchain/*
105+
du -sh libs/llamaindex/*
106+
tox -e integration-tests -c libs/colbert
107+
du -sh ~/*
108+
du -sh *
109+
du -sh libs/*
110+
du -sh libs/colbert/*
111+
du -sh libs/langchain/*
112+
du -sh libs/llamaindex/*
113+
rm -rf **/.tox
114+
du -sh libs/*
115+
du -sh libs/colbert/*
116+
du -sh libs/langchain/*
117+
du -sh libs/llamaindex/*
118+
tox -e integration-tests -c libs/llamaindex
119+
du -sh ~/*
120+
du -sh *
121+
du -sh libs/colbert/*
122+
du -sh libs/langchain/*
123+
du -sh libs/llamaindex/*
124+
rm -rf **/.tox
125+
du -sh libs/colbert/*
126+
du -sh libs/langchain/*
127+
du -sh libs/llamaindex/*
128+
tox -e integration-tests -c libs/langchain
129+
du -sh libs/colbert/*
130+
du -sh libs/langchain/*
131+
du -sh libs/llamaindex/*
91132
92133
- name: Cleanup AstraDB
93134
uses: ./.github/actions/cleanup-astra-db

‎.github/workflows/release-ragstack-colbert.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: Release RAGStack Colbert package
2+
3+
on:
4+
push:
5+
tags:
6+
- "ragstack-ai-colbert-*"
7+
8+
jobs:
9+
release:
10+
name: Release package
11+
runs-on: ubuntu-latest
12+
permissions: write-all
13+
steps:
14+
- name: Check out the repo
15+
uses: actions/checkout@v4
16+
17+
- name: "Setup: Python 3.11"
18+
uses: ./.github/actions/setup-python
19+
20+
- name: Release
21+
uses: ./.github/actions/release-package
22+
with:
23+
package-directory: "libs/colbert"
24+
pypi-token: ${{ secrets.PYPI_API_TOKEN }}
25+
pypi-test-token: ${{ secrets.PYPI_TEST_API_TOKEN }}

‎.github/workflows/release-ragstack-langchain.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: Release RAGStack LangChain package
2+
3+
on:
4+
push:
5+
tags:
6+
- "ragstack-ai-langchain-*"
7+
8+
jobs:
9+
release:
10+
name: Release package
11+
runs-on: ubuntu-latest
12+
permissions: write-all
13+
steps:
14+
- name: Check out the repo
15+
uses: actions/checkout@v4
16+
17+
- name: "Setup: Python 3.11"
18+
uses: ./.github/actions/setup-python
19+
20+
- name: Release
21+
uses: ./.github/actions/release-package
22+
with:
23+
package-directory: "libs/langchain"
24+
pypi-token: ${{ secrets.PYPI_API_TOKEN }}
25+
pypi-test-token: ${{ secrets.PYPI_TEST_API_TOKEN }}

‎.github/workflows/release-ragstack-llamaindex.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: Release RAGStack LLamaIndex package
2+
3+
on:
4+
push:
5+
tags:
6+
- "ragstack-ai-llamaindex-*"
7+
8+
jobs:
9+
release:
10+
name: Release package
11+
runs-on: ubuntu-latest
12+
permissions: write-all
13+
steps:
14+
- name: Check out the repo
15+
uses: actions/checkout@v4
16+
17+
- name: "Setup: Python 3.11"
18+
uses: ./.github/actions/setup-python
19+
20+
- name: Release
21+
uses: ./.github/actions/release-package
22+
with:
23+
package-directory: "libs/llamaindex"
24+
pypi-token: ${{ secrets.PYPI_API_TOKEN }}
25+
pypi-test-token: ${{ secrets.PYPI_TEST_API_TOKEN }}

‎.github/workflows/release-ragstack.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: Release RAGStack package
33
on:
44
push:
55
tags:
6-
- "ragstack-ai-*"
6+
- "ragstack-ai-[0-9]*"
77

88
jobs:
99
release:

0 commit comments

Comments
 (0)