[DRAFT] trulens-legacy and trulens-api

.gitignore

@@ -29,6 +29,7 @@ share/python-wheels/
 .pypirc
 MANIFEST
 *.whl
+node_modules
 
 # Notebook tests generate these files:
 imagenet_class_index.json

DEPRECATION.md

@@ -1,4 +1,4 @@
-# Deprecation Notes
+# Changes prior to the implementation of the deprecation policy
 
 ## Changes in 0.19.0
 

Makefile

@@ -1,11 +1,21 @@
-# Make targets useful for developing TruLens.
-# How to use Makefiles: https://opensource.com/article/18/8/what-how-makefile .
+# Make targets useful for developing TruLens. How to use Makefiles:
+# https://opensource.com/article/18/8/what-how-makefile . Please make sure this
+# file runs on gmake:
+# - Compunt shell syntax cannot span more than 1 line, see "lock" for example on
+#   how newline is escaped to put the for loop on the same logical line.
 
 SHELL := /bin/bash
 REPO_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
-POETRY_DIRS := $(shell find . -not -path "./dist/*" -not -path "./src/dashboard/*" -maxdepth 4 -name "*pyproject.toml" -exec dirname {} \;)
+POETRY_DIRS := $(shell find . \
+	-not -path "./dist/*" \
+	-not -path "./src/dashboard/*" \
+	-maxdepth 4 \
+	-name "*pyproject.toml" \
+	-exec dirname {} \;)
 
 # Global setting: execute all commands of a target in a single shell session.
+# Note for MAC OS, the default make is too old to support this. Brew "install
+# make" to get a newer version though it is called "gmake".
 .ONESHELL:
 
 # Create the poetry env for building website, docs, formatting, etc.
@@ -60,8 +70,8 @@ docs-serve: env-docs
 docs-serve-debug: env-docs
 	poetry run mkdocs serve -a 127.0.0.1:8000 --verbose
 
-# The --dirty flag makes mkdocs not regenerate everything when change is detected but also seems to
-# break references.
+# The --dirty flag makes mkdocs not regenerate everything when change is
+# detected but also seems to break references.
 docs-serve-dirty: env-docs
 	poetry run mkdocs serve --dirty -a 127.0.0.1:8000
 
@@ -84,27 +94,22 @@ coverage:
 # for every tested python version while those outside of static are run only for
 # the latest (supported) python version.
 test-static:
-	$(CONDA)
 	poetry run pytest --rootdir=. tests/unit/static/test_static.py
 
 # Tests in the e2e folder make use of possibly costly endpoints. They
 # are part of only the less frequently run release tests.
 
 test-deprecation:
-	$(CONDA)
 	poetry run pytest --rootdir=. tests/unit/static/test_deprecation.py
 
 # Dummy and serial e2e tests do not involve any costly requests.
 test-dummy: # has golden file
-	$(CONDA)
 	poetry run pytest --rootdir=. tests/e2e/test_dummy.py
 test-serial: # has golden file
-	$(CONDA)
 	poetry run pytest --rootdir=. tests/e2e/test_serial.py
 test-golden: test-dummy test-serial
 test-write-golden: test-write-golden-dummy test-write-golden-serial
 test-write-golden-%:
-	$(CONDA)
 	WRITE_GOLDEN=1 poetry run pytest --rootdir=. tests/e2e/test_$*.py || true
 
 # Runs required tests
@@ -119,10 +124,20 @@ test-%-allow-optional: env
 test-%-optional: env-optional
 	TEST_OPTIONAL=true make test-$*
 
-# Run the unit tests, those in the tests/unit. They are run in the CI pipeline frequently.
+# Run the unit tests, those in the tests/unit. They are run in the CI pipeline
+# frequently.
 test-unit:
 	poetry run pytest --rootdir=. tests/unit/*
 
+# Run the static unit tests only, those in the static subfolder. They are run
+# for every tested python version while those outside of static are run only for
+# the latest (supported) python version.
+test-static:
+	poetry run pytest --rootdir=. tests/unit/static/test_static.py
+
+test-deprecation:
+	TEST_OPTIONAL=1 poetry run pytest --rootdir=. tests/unit/static/test_deprecation.py
+
 # Tests in the e2e folder make use of possibly costly endpoints. They
 # are part of only the less frequently run release tests.
 test-e2e:
@@ -132,6 +147,9 @@ test-e2e:
 test-notebook:
 	poetry run pytest --rootdir=. tests/docs_notebooks/*
 
+install-wheels:
+	pip install dist/*/*.whl
+
 # Release Steps:
 ## Step: Clean repo:
 clean-dashboard:
@@ -141,8 +159,8 @@ clean: clean-dashboard
 	git clean --dry-run -fxd
 	@read -p "Do you wish to remove these files? (y/N)" -n 1 -r
 	echo
-	if [[ $$REPLY =~ ^[Yy]$$ ]]; then
-		git clean -fxd;
+	if [[ $$REPLY =~ ^[Yy]$$ ]]; then \
+		git clean -fxd; \
 	fi;
 
 ## Step: Build wheels

POLICIES.md

@@ -0,0 +1,49 @@
+# Release policies
+
+## Versioning
+
+Releases are organized in `<major>.<minor>.<patch>` style. A release is made
+about every week around tuesday-thursday. Releases increment the `minor` version
+number. Occasionally bug-fix releases occur after a weekly release. Those
+increment only the `patch` number. No releases have yet made a `major` version
+increment. Those are expected to be major releases that introduce large number
+of breaking changes.
+
+## Deprecation
+
+Changes to the public API are governed by a deprecation process in three stages.
+In the warning period of no less than _3 months_, the use of a deprecated
+package, module, or value will produce a warning but otherwise operate as
+expected. In the subsequent deprecated period of no less than _3 months_, the
+use of that component will produce an error after the deprecation message. After
+these two periods, the deprecated capability will be completely removed.
+
+!!! Info "Deprecation process"
+
+    - 3 months - warning
+
+    - 3 months - warning + error
+
+    - removal
+
+Changes that result in non-backwards compatible functionality are also reflected
+in the version numbering. In such cases, the appropriate level version change
+will occur at the introduction of the warning period.
+
+## Currently deprecating features
+
+- Starting 1.0.0, the `trulens_eval` package is being deprecated in favor of
+  `trulens` and several associated required and optional packages. See
+  [trulens_eval migration](/trulens/guides/trulens_eval_migration) for details.
+
+    - Warning period: 2024-09-01 with `trulens-eval` 1.0.0a0 -- 2024-12-01.
+      Backwards compatibility during the warning period is provided by the new
+      content of the `trulens_eval` package which provides aliases to the
+      features in their new locations. See
+      [trulens_eval](trulens/api/trulens_eval/index.md).
+
+    - Deprecated period: 2024-09-01 -- 2025-02-01 . Usage of `trulens_eval` will
+      produce warnings and errors.
+
+    - Removed expected 2024-02-01 Installation of the latest version of
+      `trulens_eval` will be an error itself.

RELEASES.md

@@ -1,12 +1,5 @@
 # Releases
 
-Releases are organized in `<major>.<minor>.<patch>` style. A release is made
-about every week around tuesday-thursday. Releases increment the `minor` version
-number. Occasionally bug-fix releases occur after a weekly release. Those
-increment only the `patch` number. No releases have yet made a `major` version
-increment. Those are expected to be major releases that introduce large number
-of breaking changes.
-
 ## 0.33.0
 
 ### What's Changed

_trulens/trulens/_bundle/__init__.py

@@ -1,7 +1,4 @@
-"""
-TruLens: Don't just vibe check your LLM app!
-
-"""
+"""TruLens: Don't just vibe check your LLM app!"""
 
 from importlib.metadata import version
 

docs/gen_ref_pages.py

@@ -1,9 +1,8 @@
 """Generate the code reference pages and navigation."""
 
-from inspect import cleandoc
 import os
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Tuple
 
 import mkdocs_gen_files
 
@@ -16,6 +15,8 @@
 
 mod_symbol = '<code class="doc-symbol doc-symbol-nav doc-symbol-module"></code>'
 
+pack_symbol = "(package)"  # <code class="doc-symbol doc-symbol-nav doc-symbol-package"></code>'
+
 
 _SPECIAL_FORMATTING = {
     "litellm": "LiteLLM",
@@ -29,7 +30,7 @@
 }
 
 
-def format_parts(parts: tuple):
+def format_parts(parts: Tuple[str, ...]) -> Tuple[str, ...]:
     if parts[0] == "trulens":
         parts = tuple(parts[1:])
 
@@ -42,41 +43,109 @@ def _format_part(idx: int, part: str):
             # internal package path starts at 1
             # trulens.core.utils -> core is nonmodule, utils is module
             seen_package_level = True
+
         if external_package and idx == 1 and part in _SPECIAL_FORMATTING:
             # trulens.providers.langchain.provider -> providers is nonmodule, langchain is package-level, provider is module
             seen_package_level = True
-            return _SPECIAL_FORMATTING[part]
+            return pack_symbol + " " + _SPECIAL_FORMATTING[part]
+
         if not seen_package_level:
             return part
+
         return f"{mod_symbol} {part}"
 
     return tuple(_format_part(i, part) for i, part in enumerate(parts))
 
 
 def write_to_gen_files(
-    parts: tuple, content: Optional[str] = None, doc_path: Optional[Path] = None
+    parts: Tuple[str, ...],
+    content: Optional[str] = None,
+    doc_path: Optional[Path] = None,
 ):
     doc_path = doc_path or Path(*parts)
 
+    print(parts)
+
     if parts[-1] == "__init__":
         parts = parts[:-1]
         doc_path = doc_path.with_name("index.md")
+
     elif parts[-1] == "__main__":
         return
 
     full_doc_path = docs_reference_path / doc_path
-    nav_parts = format_parts(parts)
+
+    if (
+        parts[0] == "trulens_eval"
+    ):  # legacy module is in the trulens-legacy package
+        nav_parts = format_parts(("legacy", *parts))
+    else:
+        nav_parts = format_parts(parts)
+
+    #    print(nav_parts)
+
     nav[nav_parts] = doc_path.as_posix()
 
     if not content:
         ident = ".".join(parts)
-        content = f"#{ident}\n::: {ident}"
+        content = f"# {ident}\n::: {ident}"
+
+        if "legacy" in parts:
+            # Show fewer details in the legacy sections.
+            content += """
+    options:
+        heading_level: 2
+        show_bases: false
+        show_root_heading: false
+        show_root_toc_entry: false
+        show_source: false
+        show_docstring_classes: false
+        show_docstring_modules: false
+        show_docstring_parameters: false
+        show_docstring_returns: false
+        show_docstring_description: true
+        show_docstring_examples: false
+        show_docstring_other_parameters: false
+        show_docstring_attributes: false
+        show_signature: false
+        separate_signature: false
+        summary: false
+        group_by_category: true
+        members_order: alphabetical
+"""
+        if "api" in parts:
+            # Show fewer details in the API sections.
+            content += """
+    options:
+        heading_level: 2
+        show_bases: false
+        show_root_heading: false
+        show_root_toc_entry: false
+        show_source: false
+        show_docstring_classes: false
+        show_docstring_modules: false
+        show_docstring_parameters: false
+        show_docstring_returns: false
+        show_docstring_description: true
+        show_docstring_examples: false
+        show_docstring_other_parameters: false
+        show_docstring_attributes: false
+        show_signature: false
+        separate_signature: false
+        summary: false
+        group_by_category: true
+        members_order: alphabetical
+"""
+    #            content = f"API {ident} [{ident}][{ident}]"
 
     with mkdocs_gen_files.open(full_doc_path, "w") as fd:
+        print("writing to", full_doc_path)
         fd.write(content)
 
 
 core_packages = [
+    "api",
+    "legacy",
     "core",
     "feedback",
     "dashboard",
@@ -91,20 +160,10 @@ def write_to_gen_files(
 packages = core_packages + provider_packages + instrument_packages
 print("Collecting from packages:", packages)
 
-# Write Index Page
-with mkdocs_gen_files.open(
-    docs_reference_path / "trulens" / "index.md", "w"
-) as fd:
-    fd.write(
-        cleandoc(
-            """
-            # TruLens API Reference
-
-            Welcome to the TruLens API Reference!
-            Use the search and navigation to explore the various modules and classes available in the TruLens library.
-            """
-        )
-    )
+nav["API Reference"] = "index.md"
+nav["providers"] = "providers/index.md"
+nav["instrument"] = "instrument/index.md"
+nav["legacy"] = "legacy/index.md"
 
 for package in packages:
     # create a nav entry for package/index.md
@@ -123,12 +182,12 @@ def write_to_gen_files(
                 continue
         if not os.path.exists(path.parent / "__init__.py"):
             print(
-                "Skipping due to missing python package: ",
+                "Skipping due to missing python module: ",
                 path.parent / "__init__.py",
             )
             continue
 
         write_to_gen_files(parts, doc_path=doc_path)
 
-    with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
-        nav_file.writelines(nav.build_literate_nav())
+with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
+    nav_file.writelines(nav.build_literate_nav())