diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..7da9c9f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,65 @@ +name: Tests +on: [push, pull_request] +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Install requirements + run: pip install flake8 pycodestyle + - name: Check syntax + run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan + + test: + needs: lint + strategy: + matrix: + include: + - ckan-version: "2.10" + ckan-image: "ckan/ckan-dev:2.10-py3.10" + solr-version: "9" + - ckan-version: "2.9" + ckan-image: "ckan/ckan-dev:2.9-py3.9" + solr-version: "8" + fail-fast: false + name: CKAN ${{ matrix.ckan-version }} + runs-on: ubuntu-latest + container: + image: ${{ matrix.ckan-image }} + options: --user root + services: + solr: + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr${{ matrix.solr-version }} + postgres: + image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:3 + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + CKAN_SOLR_URL: http://solr:8983/solr/ckan + CKAN_REDIS_URL: redis://redis:6379/1 + + steps: + - uses: actions/checkout@v4 + - name: Install requirements + run: | + pip install -r requirements.txt + pip install -r dev-requirements.txt + pip install -e . + # Replace default path to CKAN core config file with the one on the container + sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + - name: Setup extension + run: | + ckan -c test.ini db init + - name: Run tests + run: pytest --ckan-ini=test.ini --cov=ckanext.og_datatablesview --disable-warnings ckanext/og_datatablesview/tests \ No newline at end of file diff --git a/ckanext/og_datatablesview/blueprint.py b/ckanext/og_datatablesview/blueprint.py index 7e06a36..43561a9 100644 --- a/ckanext/og_datatablesview/blueprint.py +++ b/ckanext/og_datatablesview/blueprint.py @@ -13,6 +13,37 @@ ogdatatablesview = Blueprint(u'ogdatatablesview', __name__) +def format_fts_query(search_value): + u''' + Format search value for FTS query with prefix matching. + Handles compound words (with slashes/apostrophes) by using OR for parts. + ''' + if not search_value: + return u'' + space_separated = search_value.split() + processed_terms = [] + for word in space_separated: + if word: + # replace non-alphanumeric characters with FTS wildcard (_) + processed = re.sub(r'[^0-9a-zA-Z\-]+', '_', word) + if processed: + if '_' in processed: + # Compound word: use OR for parts + parts = [p for p in processed.split('_') if p] + if len(parts) > 1: + processed_terms.append(u'(' + u' | '.join([p + u':*' for p in parts]) + u')') + else: + processed_terms.append(parts[0] + u':*') + else: + # append ':*' so we can do partial FTS searches + processed_terms.append(processed + u':*') + if not processed_terms: + return u'' + if len(processed_terms) > 1: + return u' & '.join(processed_terms) + return processed_terms[0] + + def merge_filters(view_filters, user_filters_str): u''' view filters are built as part of the view, user filters @@ -94,17 +125,13 @@ def ajax(resource_view_id): v = str(request.form[u'columns[%d][search][value]' % i]) if v: k = str(request.form[u'columns[%d][name]' % i]) - # replace non-alphanumeric characters with FTS wildcard (_) - v = re.sub(r'[^0-9a-zA-Z\-]+', '_', v) - # append ':*' so we can do partial FTS searches - colsearch_dict[k] = v + u':*' + colsearch_dict[k] = format_fts_query(v) i += 1 if colsearch_dict: search_text = json.dumps(colsearch_dict) else: - search_text = re.sub(r'[^0-9a-zA-Z\-]+', '_', - search_text) + u':*' if search_text else u'' + search_text = format_fts_query(search_text) if search_text else u'' try: response = datastore_search( @@ -185,16 +212,12 @@ def filtered_download(resource_view_id): v = column[u'search'][u'value'] if v: k = column[u'name'] - # replace non-alphanumeric characters with FTS wildcard (_) - v = re.sub(r'[^0-9a-zA-Z\-]+', '_', v) - # append ':*' so we can do partial FTS searches - colsearch_dict[k] = v + u':*' + colsearch_dict[k] = format_fts_query(v) if colsearch_dict: search_text = json.dumps(colsearch_dict) else: - search_text = re.sub(r'[^0-9a-zA-Z\-]+', '_', - search_text) + u':*' if search_text else '' + search_text = format_fts_query(search_text) if search_text else '' return h.redirect_to( h.url_for( diff --git a/ckanext/og_datatablesview/tests/test_logic.py b/ckanext/og_datatablesview/tests/test_logic.py index 3814d5a..57acc27 100644 --- a/ckanext/og_datatablesview/tests/test_logic.py +++ b/ckanext/og_datatablesview/tests/test_logic.py @@ -2,6 +2,7 @@ import ckan.plugins.toolkit as toolkit from ckan.tests import factories +from ckanext.og_datatablesview.blueprint import format_fts_query @pytest.mark.usefixtures("clean_db") @@ -40,15 +41,15 @@ def test_og_datatableview_display_copy_button_success(self): resource_id=resource['id'], title='OG Data Tables', view_type='og_datatables_view', - copy_print_buttons=True + copy_print_buttons=False ) - response = p.toolkit.get_action('resource_view_show')( + response = toolkit.get_action('resource_view_show')( {'user': sysadmin.get('name')}, {'id': resource_view.get('id')} ) - assert response.get('copy_print_buttons') == True + assert response.get('copy_print_buttons') == False def test_og_datatableview_display_export_button_success(self): @@ -62,15 +63,15 @@ def test_og_datatableview_display_export_button_success(self): resource_id=resource['id'], title='OG Data Tables', view_type='og_datatables_view', - export_button=True + export_button=False ) - response = p.toolkit.get_action('resource_view_show')( + response = toolkit.get_action('resource_view_show')( {'user': sysadmin.get('name')}, {'id': resource_view.get('id')} ) - assert response.get('export_button') == True + assert response.get('export_button') == False def test_og_datatableview_custom_options_success(self): @@ -84,22 +85,22 @@ def test_og_datatableview_custom_options_success(self): resource_id=resource['id'], title='OG Data Tables', view_type='og_datatables_view', - export_button=True, - responsive=True, + export_button=False, + responsive=False, copy_print_buttons=False, - col_reorder=False + col_reorder=True ) - response = p.toolkit.get_action('resource_view_show')( + response = toolkit.get_action('resource_view_show')( {'user': sysadmin.get('name')}, {'id': resource_view.get('id')} ) - assert response.get('export_button') == True - assert response.get('responsive') == True + assert response.get('export_button') == False + assert response.get('responsive') == False assert response.get('copy_print_buttons') == False - assert response.get('col_reorder') == False + assert response.get('col_reorder') == True def test_og_datatableview_change_show_columns_success(self): @@ -139,7 +140,7 @@ def test_og_datatableview_change_show_columns_success(self): ] ) - response = p.toolkit.get_action('resource_view_show')( + response = toolkit.get_action('resource_view_show')( {'user': sysadmin.get('name')}, {'id': resource_view.get('id')} ) @@ -185,13 +186,13 @@ def test_og_datatableview_delete_resource_view_success(self): resource_view_id = resource_view.get('id') - resource_view_delete_response = p.toolkit.get_action('resource_view_delete')( + resource_view_delete_response = toolkit.get_action('resource_view_delete')( {'user': sysadmin.get('name')}, {'id': resource_view_id} ) with pytest.raises(toolkit.ObjectNotFound): - resource_view_show_response = p.toolkit.get_action('resource_view_show')( + resource_view_show_response = toolkit.get_action('resource_view_show')( {'user': sysadmin.get('name')}, {'id': resource_view_id} ) @@ -215,20 +216,72 @@ def test_og_datatableview_only_modify_one_view_success(self): view_type='og_datatables_view' ) - resource_view_update_response = p.toolkit.get_action('resource_view_update')( + resource_view_update_response = toolkit.get_action('resource_view_update')( {'user': sysadmin.get('name')}, {'id': resource_view_1.get('id'), 'description': 'Testing resource view update'}, ) - response_1 = p.toolkit.get_action('resource_view_show')( + response_1 = toolkit.get_action('resource_view_show')( {'user': sysadmin.get('name')}, {'id': resource_view_1.get('id')} ) - response_2 = p.toolkit.get_action('resource_view_show')( + response_2 = toolkit.get_action('resource_view_show')( {'user': sysadmin.get('name')}, {'id': resource_view_2.get('id')} ) assert response_1.get('description') == 'Testing resource view update' assert response_1.get('description') != response_2.get('description') + + +class TestFormatFtsQuery: + + def test_format_fts_query_empty_string(self): + assert format_fts_query('') == '' + + def test_format_fts_query_single_word(self): + assert format_fts_query('test') == 'test:*' + + def test_format_fts_query_multiple_words(self): + assert format_fts_query('test query') == 'test:* & query:*' + + def test_format_fts_query_apostrophe_compound_word(self): + # Apostrophe creates compound word - should use OR for parts + result = format_fts_query("L'est") + assert result == '(L:* | est:*)' + + def test_format_fts_query_slash_compound_word(self): + # Slash creates compound word - should use OR for parts + result = format_fts_query('Board/Village') + assert result == '(Board:* | Village:*)' + + def test_format_fts_query_mixed_words_and_compound(self): + # Mix of regular words and compound words + result = format_fts_query('Orleans Parish School Board/Village') + assert result == 'Orleans:* & Parish:* & School:* & (Board:* | Village:*)' + + def test_format_fts_query_apostrophe_in_phrase(self): + # Apostrophe in middle of phrase + result = format_fts_query("De L'est Elementary") + assert result == 'De:* & (L:* | est:*) & Elementary:*' + + def test_format_fts_query_complex_search(self): + # Complex search with multiple compound words + result = format_fts_query("Orleans Parish School Board/Village De L'est Elementary School") + assert result == 'Orleans:* & Parish:* & School:* & (Board:* | Village:*) & De:* & (L:* | est:*) & Elementary:* & School:*' + + def test_format_fts_query_special_characters(self): + # Other special characters should be replaced with underscore + result = format_fts_query('test@example.com') + assert result == '(test:* | example:* | com:*)' + + def test_format_fts_query_hyphens_preserved(self): + # Hyphens should be preserved + result = format_fts_query('test-word') + assert result == 'test-word:*' + + def test_format_fts_query_multiple_apostrophes(self): + # Multiple apostrophes + result = format_fts_query("O'Brien's") + assert result == "(O:* | Brien:* | s:*)" diff --git a/dev-requirements.txt b/dev-requirements.txt index e69de29..778004a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -0,0 +1,2 @@ +pytest-ckan +pytest-cov \ No newline at end of file