Merge pull request #54 from kbase/develop

Develop -> Master

.github/workflows/build-test-push.yml

@@ -0,0 +1,69 @@
+name: Build, Test, Push for Development Scenarios and Release
+
+on:
+  push:
+    branches: [ develop, "fix-*", "feature-*" ]
+  release:
+    types: [published]
+
+jobs:
+  docker_build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7]
+    steps:
+      - name: Check out GitHub Repo
+        uses: actions/checkout@v2
+
+      # Imports many useful environment variables
+      - uses: FranzDiebold/github-env-vars-action@v2
+
+      # Unit (ish) tests
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Pip installation
+        run: python -m pip install --upgrade pip poetry
+      - name: Poetry installation
+        run: poetry install
+      - name: Create test image
+        env:
+          TAG: ${{ env.CI_ACTION_REF_NAME }}
+        run: docker-compose build
+      - name: Run tests
+        run: scripts/run_tests
+      - name: Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.xml
+          fail_ci_if_error: true
+
+      # Build and Push (if we get here)
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to GitHub Container Registry (GHCR)
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GHCR_USERNAME }}
+          password: ${{ secrets.GHCR_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          pull: true
+          push: true
+          build-args: |
+            BUILD_CONFIG=ci
+            BUILD_DATE=${{ steps.date.outputs.date }}
+            VCS_REF=${{ github.sha }}
+            BRANCH=${{ github.ref }}
+            TAG=${{ github.ref }}
+          tags: ghcr.io/${{ github.repository_owner }}/searchapi2:${{ env.CI_ACTION_REF_NAME }}

.github/workflows/run-tests.yml → .github/workflows/test-pull-request.yml

@@ -4,8 +4,6 @@
 name: Run tests
 
 on:
-  push:
-    branches: [ develop, master ]
   pull_request:
     branches: [ "*" ]
 
@@ -25,6 +23,8 @@ jobs:
       run: python -m pip install --upgrade pip poetry
     - name: Poetry installation
       run: poetry install
+    - name: Create test image
+      run: docker-compose build
     - name: Run tests
       run: scripts/run_tests
     - name: Codecov
@@ -33,4 +33,3 @@ jobs:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: ./coverage.xml
         fail_ci_if_error: true
-

.gitignore

@@ -11,5 +11,22 @@
 coverage_report/
 .coverage
 /htmlcov/
+.pytest_cache/
 
 /coverage.xml
+
+# Testing artifacts
+# Captured container stderr and stdout
+container.err
+container.out
+
+# Development
+.vscode
+.idea
+
+# Handy place to stache temporary work, whilst keeping 
+# git unencumbered
+/_temp
+
+# Python virtual environment
+venv

CHANGELOG.md

@@ -0,0 +1,44 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [1.0.0] - 2021-04-20
+### Fixed
+- fix SCT-2930: not honoring withPublic and withPrivate
+- fix SCT-2931: maximum reported search results was 10,000 - fixed to report actual search results with > 10K.
+- fix SCT-2932: throws error if object has been deleted
+- fix SCT-2933, SCT-2956: Searchapi2 legacy endpoint uses jsonrpc 2.0 not 1.1
+- fix SCT-2937: searchapi2/legacy search_objects should be tolerant of inaccessible workspaces
+- fix SCT-2947: unauthenticated search not working correctly (no ws or narr info)
+- fix SCT-2969: not honoring object_types
+- fix SCT-2970: not narrowing search with additional terms
+- fix SCT-3001: Data-search ui  / Searchapi2 legacy endpoint generates incorrect object landing page links
+- fix SCT-3002: Searchpi2 legacy endpoint returns incorrect index and index version
+
+### Added
+- implement SCT-2966: add a "build and test" workflow for GitHub Actions which builds an image, runs tests, and pushes the resulting image to GH Container Registry.
+
+## [0.4.9] - 2020-09-11
+### Changed
+- Use the AND operator for legacy full-text search
+
+## [0.4.8] - 2020-09-09
+### Fixed
+- Set pagination parameters for the `search_workspace` method
+
+## [0.4.7] - 2020-09-03
+### Fixed
+- Include narrative info parameter setting bug
+
+## [0.4.6] - 2020-08-31
+### Fixed
+- Fixed the highlight fields for legacy search
+
+## [0.4.5] - 2020-08-25
+### Fixed
+- Prevent removal of trailing slash in configured user profile URL
+- Fix format string in error

CODEOWNERS

@@ -1 +1 @@
-* @jayrbolton @slebras
+* @slebras @scanon @eapearson

Dockerfile

@@ -7,30 +7,30 @@ ARG BRANCH=develop
 ENV DOCKERIZE_VERSION v0.6.1
 
 # Install dockerize
-RUN apk --update add --virtual build-dependencies curl tar gzip && \
+RUN apk --update add --no-cache --virtual build-dependencies curl tar gzip && \
     curl -o dockerize.tar.gz \
-      https://raw.githubusercontent.com/kbase/dockerize/master/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \
+    https://raw.githubusercontent.com/kbase/dockerize/master/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \
     tar -C /usr/local/bin -xvzf dockerize.tar.gz && \
     rm dockerize.tar.gz && \
-    apk del build-dependencies
+    apk del --no-cache build-dependencies
 
 # Dockerize related labels
 LABEL org.label-schema.build-date=$BUILD_DATE \
-      org.label-schema.vcs-url="https://github.com/kbaseincubator/search_api_deluxe" \
-      org.label-schema.vcs-ref=$VCS_REF \
-      org.label-schema.schema-version="1.0.0-rc1" \
-      us.kbase.vcs-branch=$BRANCH \
-      maintainer="KBase Team"
+    org.label-schema.vcs-url="https://github.com/kbaseincubator/search_api_deluxe" \
+    org.label-schema.vcs-ref=$VCS_REF \
+    org.label-schema.schema-version="1.0.0-rc1" \
+    us.kbase.vcs-branch=$BRANCH \
+    maintainer="KBase Team"
 
 WORKDIR /app
 
 # Install dependencies
 COPY pyproject.toml poetry.lock /app/
-RUN apk --update add --virtual build-dependencies libffi-dev libressl-dev musl-dev python3-dev build-base && \
+RUN apk --update add --no-cache --virtual build-dependencies libffi-dev libressl-dev musl-dev python3-dev build-base git rust cargo && \
     pip install --upgrade pip poetry==1.0.9 && \
     poetry config virtualenvs.create false && \
     poetry install --no-dev --no-interaction --no-ansi && \
-    apk del build-dependencies
+    apk del --no-cache build-dependencies
 
 COPY . /app
 

LICENSE.md

@@ -1,4 +1,4 @@
-Copyright (c) 2019 The KBase Project and its Contributors
+Copyright (c) 2021 The KBase Project and its Contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

Makefile

@@ -0,0 +1,18 @@
+# Makefile for search_api2
+.PHONY: all test docs
+
+test:
+	sh scripts/run_tests
+
+build-dev-images:
+	@echo Building integration test images...
+	sh scripts/build-integration-test-images.sh
+	@echo Integration test images built
+
+integration-tests: 
+	@echo Running Integration Tests...
+	sh scripts/run_integration_tests
+
+run-dev-server:
+	@echo Starting dev server...
+	sh scripts/run-dev-server.sh

README.md

@@ -9,16 +9,16 @@ This service has two JSON-RPC 2.0 endpoints:
 * `/legacy` - mirrors the old Java JSON-RPC 1.1 methods
 * `/rpc` - newer API using the Elasticsearch Query DSL
 
-The JSON-Schemas for the legacy methods can be found in `legacy-schema.yaml`
+The JSON-Schemas for the legacy methods can be found in `src/search1_rpc/schemas`
 
-The JSON-Schemas for the newer methods (/rpc) can be found in `rpc-schema.yaml`
+The JSON-Schemas for the newer methods (`/rpc`) can be found in `rpc-schema.yaml`
 
 ### Documents and indexes
 
 The [search configuration file](https://github.com/kbase/index_runner_spec/blob/master/config.yaml) details all of the indexes and document types found in the KBase Elasticsearch database.
 
 * `ws_type_to_indexes` shows which KBase workspace types (without versions) map to which unversioned index names
-* `ws_subobjects` is a list of indexes that represent KBase "subobjects", such as genome features, that don't have their own dedicated type in the workspace, but do have a dedicated index in Elasticsearch.
+* `ws_subobjects` is a list of indexes that represent KBase "sub objects", such as genome features, that don't have their own dedicated type in the workspace, but do have a dedicated index in Elasticsearch.
 * `global_mappings` are Elasticsearch type definitions that are reused in many of the indexes below.
 * `latest_versions` map the unversioned index names to the versioned index names that represent the latest type mapping version.
 * `aliases` is a list of Elasticsearch index aliases to a list of index names. These are all searchable as index names.
@@ -31,8 +31,11 @@ The [search configuration file](https://github.com/kbase/index_runner_spec/blob/
 * `-32003` - Elasticsearch response error
 * `-32004` - User profile service response error
 * `-32005` - Unknown workspace type
+* `-32006` - Access group missing
+* `-32007` - User profile missing
 
-### <url>/rpc
+
+### `<url>/rpc`
 
 Uses [JSON RPC 2.0 format](https://www.jsonrpc.org/specification).
 
@@ -80,7 +83,7 @@ Show the names of all indexes, and show what aliases stand for what indexes.
 
 ### <url>/legacy
 
-A JSON-RPC 2.0 API that mimics the legacy Java server, [found here](https://github.com/kbase/KBaseSearchEngin://github.com/kbase/KBaseSearchEngine). Refer to the `legacy-schema.yaml` file for a reference on the method parameter types.
+A JSON-RPC 1.1 API that mimics the legacy Java server, [found here](https://github.com/kbase/KBaseSearchEngin://github.com/kbase/KBaseSearchEngine). Refer to the `src/search1_rpc/schemas` file for a reference on the method parameter types.
 
 ## Development
 
@@ -107,18 +110,8 @@ volume for the Elasticsearch service to see the changes. You can do this with `d
 
 ### Running the integration tests
 
-Under `tests/integration`, there is a module of integration tests that run against CI (KBase staging server).
+Under `tests/integration`, there is a set of integration tests that run against CI (KBase staging server).
 
 These do not run in our CI workflow, but are only run manually/locally.
 
-You need to ssh tunnel to get CI Elasticsearch available locally:
-
-```sh
-ssh -L 9500:<elasticsearch_ip>:9500 [email protected]
-```
-
-Then you can run the integration test script:
-
-```sh
-sh scripts/run_integration_tests.sh
-```
+Please see the [integration testing docs](docs/integration-testing.md) for instructions and further information.

VERSION

@@ -1 +1 @@
-0.4.2
+1.0.0

docker-compose.yaml

@@ -6,6 +6,7 @@ services:
 
   # For running the python consumers
   web:
+    image: searchapi2:${TAG:-dev}
     build:
       context: .
     depends_on:

docs/ez-guide.md

@@ -0,0 +1,82 @@
+# An E-Z Guide
+
+This is a set of tasks which worked well for me on macOS.
+
+## Unit Testing
+
+### Host dependencies
+
+You will need to have, at a minimum:
+
+- make
+- python 3.7
+- docker
+
+### Set up python
+
+Install virtual environment for python:
+
+```sh
+python -m venv venv
+source  venv/bin/activate
+python -m pip install --upgrade pip
+```
+
+Install `poetry`:
+
+```sh
+pip install poetry
+```
+
+Unit tests are run locally, so need to install all python dependencies:
+
+```sh
+poetry install
+```
+
+### Run Tests
+
+> TODO: should be able to run unit tests in a container, to avoid the need for any host-level installs.
+
+Run the tests!
+
+This will run all the unit tests plus associated code quality evaluations.
+
+```sh
+make test
+```
+
+or for coverage
+
+```sh
+make test-coverage
+```
+
+> Note: Ensure that https://ci.kbase.us is accessible from the host machine; some unit tests require this (and should not be unit tests!)
+
+To run tests in a given directory or individual test modules:
+
+```sh
+WORKSPACE_URL="http://localhost:5555/ws" PYTHONPATH=. poetry run pytest -vv tests/unit/PATH
+```
+
+e.g. to run all the `es_client` tests:
+
+```sh
+WORKSPACE_URL="http://localhost:5555/ws" PYTHONPATH=. poetry run pytest -vv tests/unit/es_client
+```
+
+## Integration Testing
+
+See [Integration Testing](integration-testing.md)
+
+
+## Using with kbase-ui
+
+This workflow is very handy for working on both the search api back end and search tool front ends.
+
+```
+IP="<IP HERE>" SSHHOST="login1.berkeley.kbase.us" SSHUSER="<KBASE DEV USERNAME>" SSHPASS="<KBASE DEV PWD>" make run-dev-server
+```
+
+> TODO: complete this doc!