From 27bcebd43a6fb13caf8007e3c66b407b1fb9e7b1 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 3 Mar 2026 12:36:42 -0500 Subject: [PATCH 1/3] chore: alpha cleanup and beta prep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dead code removal (~2,500 lines): - Deleted selection/, validation/, visualization/, pipeline/ modules - Deleted config/feature_config.py, config/validation.py, core/deprecation.py - Deleted labeling backward-compat shims (core.py, barriers.py, barrier_utils.py) - Removed all __getattr__ error traps, D06 aliases, deprecated params - Cleaned bar samplers: removed initial_expectation/initial_run_expectation - Cleaned mom(): timeperiod → period New tests: - +58 comprehensive volatility tests (all 11 non-TA-Lib estimators + Bollinger) CI improvements: - ta-lib separated from dev deps, test job uses --extra ta - 57 perf tests marked @pytest.mark.perf, excluded from default runs - Removed 6 calendar test deselections (bugs fixed) - Removed mypy config block and optional extra Bug fixes: - EquityCalendar._next_basic_open: returned past time during market hours - EquityCalendar._previous_basic_close: returned future time during market hours Docs: - Created CHANGELOG.md (full alpha history a3-a11) - Updated user guides for features, labeling, bars - Added dataset-builder, discovery, preprocessing, fractional-differencing guides --- .github/workflows/ci.yml | 15 +- CHANGELOG.md | 130 +++ README.md | 11 + docs/audit/book-integration-audit.md | 190 ++++ docs/getting-started/quickstart.md | 8 +- docs/index.md | 11 + docs/user-guide/bars.md | 2 + docs/user-guide/dataset-builder.md | 203 ++++ docs/user-guide/discovery.md | 162 +++ docs/user-guide/features.md | 441 ++++++-- docs/user-guide/fractional-differencing.md | 188 ++++ docs/user-guide/labeling.md | 508 +++++++++- docs/user-guide/ml-readiness.md | 2 + docs/user-guide/preprocessing.md | 171 ++++ pyproject.toml | 22 +- src/ml4t/engineer/__init__.py | 10 +- src/ml4t/engineer/bars/imbalance.py | 29 - src/ml4t/engineer/bars/run.py | 38 - src/ml4t/engineer/bars/vectorized.py | 4 - src/ml4t/engineer/config/__init__.py | 137 +-- src/ml4t/engineer/config/feature_config.py | 946 ----------------- src/ml4t/engineer/config/labeling.py | 13 - src/ml4t/engineer/config/validation.py | 281 ------ src/ml4t/engineer/core/calendars/equity.py | 30 +- src/ml4t/engineer/core/deprecation.py | 135 --- src/ml4t/engineer/features/momentum/mom.py | 14 +- src/ml4t/engineer/labeling/__init__.py | 13 - src/ml4t/engineer/labeling/barrier_utils.py | 10 - src/ml4t/engineer/labeling/barriers.py | 9 - src/ml4t/engineer/labeling/core.py | 15 - src/ml4t/engineer/pipeline/__init__.py | 8 - src/ml4t/engineer/pipeline/engine.py | 259 ----- src/ml4t/engineer/preprocessing.py | 4 +- src/ml4t/engineer/selection/__init__.py | 23 - src/ml4t/engineer/validation/README.md | 59 -- src/ml4t/engineer/validation/__init__.py | 13 - src/ml4t/engineer/validation/cv.py | 21 - src/ml4t/engineer/visualization/__init__.py | 16 - src/ml4t/engineer/visualization/summary.py | 78 -- tests/bars/test_imbalance_bars.py | 6 - tests/bars/test_run_bars.py | 22 - tests/bars/test_vectorized_bars.py | 14 +- tests/core/test_calendars.py | 50 +- .../test_volatility_comprehensive.py | 950 ++++++++++++++++++ tests/test_ad.py | 1 + tests/test_adosc.py | 1 + tests/test_adxr.py | 1 + tests/test_avgdev.py | 1 + tests/test_avgprice.py | 1 + tests/test_bars.py | 62 +- tests/test_bars_specialized.py | 19 +- tests/test_cmo.py | 1 + tests/test_config_system.py | 51 - tests/test_directional_indicators.py | 1 + tests/test_dm_indicators.py | 1 + tests/test_fdiff.py | 38 - tests/test_imi.py | 1 + tests/test_integration_pipeline.py | 430 -------- tests/test_kama.py | 1 + tests/test_labeling.py | 53 +- tests/test_linearreg.py | 1 + tests/test_linearreg_family.py | 1 + tests/test_math_operators.py | 1 + tests/test_medprice.py | 1 + tests/test_midpoint.py | 1 + tests/test_midprice.py | 1 + tests/test_natr.py | 1 + tests/test_new_indicators.py | 12 +- tests/test_optimized_indicators.py | 1 + tests/test_performance.py | 8 + tests/test_pipeline_engine.py | 366 ------- tests/test_risk.py | 1 + tests/test_roc_variants.py | 1 + tests/test_stochf.py | 1 + tests/test_t3.py | 1 + tests/test_talib_accuracy.py | 3 + tests/test_talib_p0_indicators.py | 11 +- tests/test_timezone_handling.py | 10 +- tests/test_trange.py | 1 + tests/test_trix.py | 1 + tests/test_tsf.py | 1 + tests/test_typprice.py | 1 + tests/test_ultosc.py | 1 + tests/test_var.py | 1 + tests/test_wclprice.py | 1 + tests/visualization/__init__.py | 1 - tests/visualization/test_summary.py | 51 - uv.lock | 126 +-- 88 files changed, 3031 insertions(+), 3510 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 docs/audit/book-integration-audit.md create mode 100644 docs/user-guide/dataset-builder.md create mode 100644 docs/user-guide/discovery.md create mode 100644 docs/user-guide/fractional-differencing.md create mode 100644 docs/user-guide/preprocessing.md delete mode 100644 src/ml4t/engineer/config/feature_config.py delete mode 100644 src/ml4t/engineer/config/validation.py delete mode 100644 src/ml4t/engineer/core/deprecation.py delete mode 100644 src/ml4t/engineer/labeling/barrier_utils.py delete mode 100644 src/ml4t/engineer/labeling/barriers.py delete mode 100644 src/ml4t/engineer/labeling/core.py delete mode 100644 src/ml4t/engineer/pipeline/__init__.py delete mode 100644 src/ml4t/engineer/pipeline/engine.py delete mode 100644 src/ml4t/engineer/selection/__init__.py delete mode 100644 src/ml4t/engineer/validation/README.md delete mode 100644 src/ml4t/engineer/validation/__init__.py delete mode 100644 src/ml4t/engineer/validation/cv.py delete mode 100644 src/ml4t/engineer/visualization/__init__.py delete mode 100644 src/ml4t/engineer/visualization/summary.py create mode 100644 tests/features/volatility/test_volatility_comprehensive.py delete mode 100644 tests/test_integration_pipeline.py delete mode 100644 tests/test_pipeline_engine.py delete mode 100644 tests/visualization/__init__.py delete mode 100644 tests/visualization/test_summary.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3275317..b2f98a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,19 +72,18 @@ jobs: - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} + - name: Install TA-Lib C library + run: | + sudo apt-get update + sudo apt-get install -y libta-lib0-dev + - name: Install dependencies - run: uv sync --dev + run: uv sync --dev --extra ta - name: Run tests run: | set +e - uv run pytest tests/ -v --tb=short -x --no-cov \ - --deselect tests/core/test_calendars.py::TestNextOpenPreviousClose::test_next_open_from_market_hours \ - --deselect tests/core/test_calendars.py::TestNextOpenPreviousClose::test_previous_close_from_market_hours \ - --deselect tests/core/test_calendars.py::TestNextOpenPreviousClose::test_previous_close_from_before_open \ - --deselect tests/core/test_calendars.py::TestNextOpenPreviousClose::test_previous_close_skips_weekend \ - --deselect tests/core/test_calendars.py::TestSessionsBetween::test_sessions_between_same_week \ - --deselect tests/core/test_calendars.py::TestCalendarIntegration::test_timezone_aware_workflow + uv run pytest tests/ -v --tb=short -x --no-cov PYTEST_EXIT=$? if [ $PYTEST_EXIT -eq 0 ] || [ $PYTEST_EXIT -eq 5 ]; then echo "Tests passed (exit code: $PYTEST_EXIT)" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..dc0bb1d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,130 @@ +# Changelog + +All notable changes to ml4t-engineer are documented in this file. + +Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Removed +- Dead modules: `selection/`, `validation/`, `visualization/`, `pipeline/` +- Diagnostic config classes (`feature_config.py`) — moved to ml4t-diagnostic +- Deprecation machinery (`core/deprecation.py`, deprecated params in bar samplers and `mom()`) +- Backward-compatibility shims in labeling module +- `[tool.mypy]` config (migrated to ty) + +### Added +- Comprehensive volatility tests (58 new tests covering all 11 non-TA-Lib estimators) +- `perf` pytest marker — performance benchmarks excluded from default runs, available via `pytest -m perf` + +### Changed +- TA-Lib moved from dev dependency group to `[ta]` optional extra (fixes CI for lint/typecheck jobs) +- `mom()` parameter renamed: `timeperiod` → `period` (consistency with other indicators) +- Bar sampler constructors: removed `initial_expectation` / `initial_run_expectation` params + +## [0.1.0a11] - 2026-03-03 + +### Changed +- API hardening and correctness fixes for beta preparation +- Labeling leakage gap closed: data sorted chronologically before all label computations +- Public API aligned with documentation + +## [0.1.0a10] - 2026-02-28 + +### Fixed +- Labeling leakage gap: ensured chronological sorting in all labeling functions +- Public API documentation alignment + +## [0.1.0a9] - 2026-02-28 + +### Fixed +- `__version__` sourced from generated version metadata instead of hardcoded string + +## [0.1.0a8] - 2026-02-27 + +### Added +- GitHub Actions CI workflow (lint, typecheck, test matrix, build) +- Release workflow with OIDC trusted publishing +- Ecosystem diagrams in README + +### Changed +- Removed outcome module (migrated to ml4t-diagnostic) +- Feature count: 120 features across 10 categories +- Standardized labeling API on `LabelingConfig`-first pattern + +### Fixed +- Normalized metadata for 4 features (33 → 37 normalized) +- ty type checking rules and CI configuration +- Numba cleanup crash workaround for Python 3.13 + +## [0.1.0a7] - 2026-01-20 + +### Added +- Time-based duration strings for labeling horizons (`"1h"`, `"4h"`, `"1d"`) +- `fixed_time_horizon_labels()` accepts `horizon="1h"` +- `triple_barrier_labels()` accepts `max_holding_period="1h"` +- `rolling_percentile_binary_labels()` accepts time-based horizon/lookback +- 51 new tests for time-based horizons + +### Fixed +- Chronological sorting in `triple_barrier_labels`, `trend_scanning_labels`, + `fixed_time_horizon_labels`, and `rolling_percentile_binary_labels` +- dtype-based timestamp detection (replaces name matching) + +## [0.1.0a6] - 2026-01-18 + +### Added +- Validation infrastructure with AFML and mlfinpy reference tests +- 86 validation tests (AFML formulas + mlfinpy comparison) +- Triple barrier, meta-labeling, sample weights validated at 1e-10 tolerance + +### Fixed +- Triple barrier edge cases +- Multiple drift detection bugs +- Tuple syntax for isinstance type checks + +## [0.1.0a5] - 2026-01-14 + +### Added +- `get_agent_docs()` for AI agent discoverability +- Hierarchical AGENT.md navigation files +- AGENT.md files included in wheel builds + +### Fixed +- `variance_ratio` Int64 bug + +## [0.1.0a4] - 2026-01-08 + +### Fixed +- Synced missing modules from development workspace + +## [0.1.0a3] - 2026-01-04 + +Initial public alpha release. + +### Added +- 120 feature functions across 10 categories (momentum, trend, volatility, + volume, microstructure, ML, risk, cycle, pattern, statistics) +- 60 indicators validated against TA-Lib at 1e-6 tolerance +- Triple-barrier labeling system (De Prado AFML) +- ATR-adjusted barriers, fixed horizon, trend scanning, percentile labels +- Meta-labeling and sample uniqueness (sequential bootstrap) +- Alternative bar types: volume, dollar, tick, imbalance, run bars +- Polars-native with Numba JIT compilation +- `compute_features()` pipeline with dependency resolution +- `FeatureCatalog` for feature discovery and metadata +- `LabelingConfig` with Pydantic v2 serialization +- `MLDatasetBuilder` for dataset construction +- `PreprocessingPipeline` for feature transformation + +[Unreleased]: https://github.com/stefan-jansen/ml4t-engineer/compare/v0.1.0a11...HEAD +[0.1.0a11]: https://github.com/stefan-jansen/ml4t-engineer/compare/v0.1.0a10...v0.1.0a11 +[0.1.0a10]: https://github.com/stefan-jansen/ml4t-engineer/compare/v0.1.0a9...v0.1.0a10 +[0.1.0a9]: https://github.com/stefan-jansen/ml4t-engineer/compare/v0.1.0a8...v0.1.0a9 +[0.1.0a8]: https://github.com/stefan-jansen/ml4t-engineer/compare/v0.1.0a7...v0.1.0a8 +[0.1.0a7]: https://github.com/stefan-jansen/ml4t-engineer/compare/v0.1.0a6...v0.1.0a7 +[0.1.0a6]: https://github.com/stefan-jansen/ml4t-engineer/compare/v0.1.0a5...v0.1.0a6 +[0.1.0a5]: https://github.com/stefan-jansen/ml4t-engineer/compare/v0.1.0a4...v0.1.0a5 +[0.1.0a4]: https://github.com/stefan-jansen/ml4t-engineer/compare/v0.1.0a3...v0.1.0a4 +[0.1.0a3]: https://github.com/stefan-jansen/ml4t-engineer/releases/tag/v0.1.0a3 diff --git a/README.md b/README.md index 656fda3..64618a2 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,17 @@ dbars = DollarBarSampler(dollar_threshold=1_000_000).sample(tick_data) ibars = TickImbalanceBarSampler(expected_imbalance=100).sample(tick_data) ``` +## Documentation + +- [Features](docs/user-guide/features.md) - 120 technical indicators across 11 categories +- [Labeling](docs/user-guide/labeling.md) - 7 labeling methods for supervised learning +- [Alternative Bars](docs/user-guide/bars.md) - Information-driven bar sampling +- [Feature Discovery](docs/user-guide/discovery.md) - Registry, catalog, and search +- [Fractional Differencing](docs/user-guide/fractional-differencing.md) - Memory-preserving stationarity +- [ML-Readiness](docs/user-guide/ml-readiness.md) - Normalized features and preprocessing +- [Preprocessing](docs/user-guide/preprocessing.md) - Scalers and leakage prevention +- [Dataset Builder](docs/user-guide/dataset-builder.md) - Leakage-safe train/test preparation + ## Technical Characteristics - **Polars-native**: All computations use Polars expressions diff --git a/docs/audit/book-integration-audit.md b/docs/audit/book-integration-audit.md new file mode 100644 index 0000000..19c3871 --- /dev/null +++ b/docs/audit/book-integration-audit.md @@ -0,0 +1,190 @@ +# Book Integration Audit: ml4t-engineer + +*Audit date: 2026-03-03 | Library version: v0.1.0a11 | Book: Machine Learning for Trading, 3rd Edition* + +## Executive Summary + +ml4t-engineer's core value proposition is validated by heavy book usage: 120 features, 7 labeling methods, and 11 bar samplers are all exercised in chapters 3, 7-9 and all 9 case studies. Several high-quality modules (MLDatasetBuilder, FeatureCatalog.search()) previously had zero book exposure but are now showcased in Ch7 NB10. The pipeline engine and DuckDB store are honestly low-value. + +--- + +## Usage Matrix: Chapter x Module + +### Feature Computation (`compute_features`) + +| Chapter / Case Study | Notebook | Features Used | Notes | +|---------------------|----------|---------------|-------| +| Ch7 | `10_ml4t_library_ecosystem.py` | rsi, sma, ema, atr, macd, bollinger_bands | Registry tour, 3 input formats, MLDatasetBuilder | +| Ch8 | `01_price_volume_features.py` | momentum (31), trend (10), volatility (15) | Core feature teaching | +| Ch8 | `02_microstructure_features.py` | microstructure (12) | Kyle Lambda, VPIN, Amihud | +| Ch8 | `03_structural_cross_instrument_features.py` | cross-asset (10) | beta_to_market, correlations | +| Ch8 | `04_fundamentals_macro_calendar.py` | ML features, calendar | Lag, encodings, macro features | +| Ch9 | `02_structural_breaks.py` | statistics | Structural break detection | +| Ch9 | `03_fractional_differencing.py` | ffdiff (4) | Fractional differencing + ADF | +| Ch9 | `05_spectral_features.py` | ML features | Spectral, FFT | +| Ch9 | `08_garch_volatility.py` | volatility (15) | GARCH, EWMA, realized vol | +| Ch9 | `09_har_rough_volatility.py` | volatility | HAR model features | +| Ch9 | `11_hmm_regimes.py` | regime (6) | Hurst, HMM state probabilities | +| Ch9 | `13_regime_as_feature.py` | regime (6) | Regime encoding as features | +| Ch9 | `14_panel_features.py` | cross-asset (10) | Cross-sectional panel features | +| ETFs | `03_features.py`, `04_temporal.py` | momentum, volatility, volume, ffdiff | Full pipeline | +| US Equities Panel | `03_features.py`, `04_temporal.py` | momentum, volatility, ffdiff | Full pipeline | +| CME Futures | `02_labels.py`, `03_features.py` | momentum, volatility, atr | Futures-specific | + +### Labeling Methods + +| Chapter / Case Study | Notebook | Method | Config Style | +|---------------------|----------|--------|--------------| +| Ch7 | `03_label_methods.py` | triple_barrier_labels | LabelingConfig.triple_barrier() | +| Ch7 | `03_label_methods.py` | rolling_percentile_binary_labels | Direct call | +| Ch7 | `03_label_methods.py` | trend_scanning_labels | Direct call | +| Ch7 | `03_label_methods.py` | meta_labels + compute_bet_size | Meta-labeling workflow | +| Ch7 | `03_label_methods.py` | sequential_bootstrap | Sample weighting | +| CME Futures | `02_labels.py` | atr_triple_barrier_labels | LabelingConfig.atr_barrier() | +| ETFs | `02_labels.py` | rolling_percentile_binary_labels | Direct call | +| US Equities Panel | `02_labels.py` | triple_barrier_labels | LabelingConfig | +| All case studies | `02_labels.py` | fixed_time_horizon_labels | Direct call | + +### Alternative Bar Sampling + +| Chapter | Notebook | Sampler | Notes | +|---------|----------|---------|-------| +| Ch3 | `08_itch_bar_sampling.py` | TickBarSampler, VolumeBarSampler, DollarBarSampler | ITCH tick data | +| Ch3 | `10_itch_information_bars.py` | TickImbalanceBarSampler, FixedTickImbalanceBarSampler | Information-driven bars | +| Ch3 | `13_databento_bar_sampling.py` | Bar sampling on Databento data | Alternative data source | + +### Feature Discovery & Registry + +| Chapter | Notebook | API Used | +|---------|----------|----------| +| Ch7 | `10_ml4t_library_ecosystem.py` | get_registry(), list_all(), get(), list_by_category() | +| Ch7 | `10_ml4t_library_ecosystem.py` | feature_catalog.search(), feature_catalog.list(), describe() | +| Ch7 | `10_ml4t_library_ecosystem.py` | compute_features (3 formats: list, dict, YAML) | + +### MLDatasetBuilder & Preprocessing + +| Chapter | Notebook | API Used | +|---------|----------|----------| +| Ch7 | `10_ml4t_library_ecosystem.py` | create_dataset_builder, train_test_split, scaler="robust" | +| Ch7 | `10_ml4t_library_ecosystem.py` | LabelingConfig.to_yaml(), from_yaml() | +| Ch7 | `02_preprocessing_pipeline.py` | StandardScaler, split-aware preprocessing | + +--- + +## Book Chapter Structure (Actual) + +| Chapter | Directory | Notebooks | Primary ml4t-engineer Usage | +|---------|-----------|-----------|----------------------------| +| Ch3 | `03_market_microstructure/` | 17 | bars module | +| Ch7 | `07_defining_learning_task/` | 10 | labeling, registry, dataset builder | +| Ch8 | `08_feature_engineering/` | 8 + meta | features (all categories) | +| Ch9 | `09_time_series_analysis/` | 14 + meta | volatility, regime, ffdiff, cross-asset | + +### Case Study Structure (Standard Pattern) + +All 9 case studies follow the same 18-file pattern: + +| Step | File | ml4t-engineer Usage | +|------|------|---------------------| +| Setup | `01_setup.py` | — | +| Labels | `02_labels.py` | `atr_triple_barrier_labels`, `rolling_percentile_binary_labels`, `fixed_time_horizon_labels` | +| Features | `03_features.py` | `compute_features`, individual feature functions | +| Temporal | `04_temporal.py` | `ffdiff`, walk-forward CV | +| Evaluation | `05_evaluation.py` | — (ml4t-diagnostic) | +| Models | `06-13_*.py` | — | +| Backtest | `14_backtest.py` | — (ml4t-backtest) | + +--- + +## Feature Triage + +### Heavily Used (Core Value) + +| Module | Lines | Book Coverage | Confidence | Action | +|--------|-------|---------------|------------|--------| +| 120 features (10 categories) | ~8,000 | Ch8 (8 notebooks), Ch9 (14 notebooks), 9 case studies | 59 TA-Lib validated | Keep, document well | +| 7 labeling methods | ~2,000 | Ch7 NB03, all 9 case study `02_labels.py` | AFML validated | Keep, document well | +| 11 bar samplers | ~2,000 | Ch3 (3 notebooks) | Production-ready | Keep, document well | +| ffdiff module | 383 | Ch9 NB03, ETFs/Equities `04_temporal.py` | Unique value | Keep, dedicated guide | +| LabelingConfig | 467 | Ch7 NB03, all case studies | API surface | Keep, document well | +| Registry/Catalog | ~650 | Ch7 NB10 | Discovery | Keep, dedicated guide | +| MLDatasetBuilder | 638 | Ch7 NB10 (newly added) | Leakage-safe prep | Keep, dedicated guide | + +### Honestly Low-Value + +| Module | Lines | Assessment | Recommended Action | +|--------|-------|------------|-------------------| +| Pipeline engine | ~300 | `compute_features` already handles dependency ordering. Thin DAG wrapper adds little. | Label "Advanced" | +| Store (DuckDB) | ~500 | No adoption path, no book usage, no clear user need. | Label "Experimental" | +| FeatureSelector | stub | Correctly moved to ml4t-diagnostic. Stub remains as migration aid. | Keep stub, document redirect | + +--- + +## Case Studies NOT Using ml4t-engineer + +These case studies implement features manually. This is **correct** in most cases: + +| Case Study | Reason for Manual Implementation | Library Overlap | +|-----------|----------------------------------|-----------------| +| Crypto Perps Funding | Domain-specific funding rate features | None — inline appropriate | +| S&P 500 Options / Option Analytics | Greeks, IV surfaces — specialized derivatives analytics | None — out of scope | +| US Firm Characteristics | Accounting ratios from financial statements | None — out of scope | +| NASDAQ-100 Microstructure | Kyle's Lambda, Amihud, VPIN implemented manually for pedagogy | **High** — all in library (callout added) | +| FX Pairs | Garman-Klass volatility, momentum features | **Partial** — some in library (callout added) | + +--- + +## Cross-Reference: Book Notebooks Using ml4t.engineer + +### Direct imports (`from ml4t.engineer`) + +| File | Imports | Status | +|------|---------|--------| +| `07_defining_learning_task/code/10_ml4t_library_ecosystem.py` | compute_features, get_registry, feature_catalog, create_dataset_builder, LabelingConfig | Working | +| `07_defining_learning_task/code/03_label_methods.py` | LabelingConfig, 7 labeling functions | Working (migrated from BarrierConfig, un-skipped) | +| `07_defining_learning_task/code/04_minimum_favorable_adverse_excursion.py` | LabelingConfig | Working (migrated from BarrierConfig, un-skipped) | +| `08_feature_engineering/code/01_price_volume_features.py` | ml4t.engineer.features.volatility, momentum, trend | Working | +| `09_time_series_analysis/code/08_garch_volatility.py` | ml4t.engineer.features.volatility (6 functions) | Working | +| All case study `02_labels.py` | ml4t.engineer.labeling (atr_triple_barrier_labels etc.) | Working | +| All case study `03_features.py` | ml4t.engineer.features (momentum, volatility, regime, trend) | Working | + +### Indirect usage (via `utils/label_functions.py`) + +Some case study `02_labels.py` files use standalone label utility wrappers that mirror the ml4t.engineer API. These are isolated from API changes but are less idiomatic. + +--- + +## Documentation Coverage + +| User Guide Page | Lines | Book Reference | Status | +|----------------|-------|----------------|--------| +| `labeling.md` | 522 | Ch7 `03_label_methods.py`, CME `02_labels.py`, ETFs `02_labels.py` | Complete | +| `features.md` | 388 | Ch8 NB01-04, Ch9 NB08-14, ETFs/Equities/CME `03_features.py` | Complete | +| `bars.md` | 405 | Ch3 `08_itch_bar_sampling.py`, `10_itch_information_bars.py`, `13_databento_bar_sampling.py` | Complete | +| `ml-readiness.md` | 178 | Ch8 `01_price_volume_features.py` | Complete | +| `discovery.md` | 162 | Ch7 `10_ml4t_library_ecosystem.py` | Complete | +| `fractional-differencing.md` | 188 | Ch9 `03_fractional_differencing.py`, ETFs/Equities `04_temporal.py` | Complete | +| `preprocessing.md` | 171 | Ch7 `02_preprocessing_pipeline.py` | Complete | +| `dataset-builder.md` | 201 | Ch7 `10_ml4t_library_ecosystem.py` | Complete | + +--- + +## Value Assessment + +### What ml4t-engineer does well + +1. **Feature computation is the clear winner**: 120 features, validated, fast, config-driven. Used in 30+ notebooks across 8 chapters and 9 case studies. +2. **Labeling methods are comprehensive**: All 7 AFML methods implemented, validated, calendar-aware. Used in every case study. +3. **Bar sampling is uniquely valuable**: No other Python library provides production-quality imbalance bars with threshold spiral warnings. +4. **Registry/discovery is elegant**: Metadata-driven feature selection with TA-Lib compatibility flags and normalization status. +5. **MLDatasetBuilder fills a real gap**: Leakage-safe dataset prep with CV integration — now demonstrated in Ch7 NB10. + +### What should be scoped honestly + +1. **Pipeline engine**: `compute_features` already does dependency ordering. The Pipeline class adds a thin DAG wrapper that few users need. Document as "Advanced". +2. **DuckDB Store**: No user demand, no book usage. Keep but label experimental. +3. **Cross-asset features (8 of 10 unused in book)**: Strong implementations but limited coverage. Only `beta_to_market` and `rolling_correlation` are commonly needed. + +--- + +*This audit was used to drive the user guide expansion and book notebook updates for v0.1.0a11.* diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 2a6db1f..e24dfda 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -97,6 +97,10 @@ print(info) ## Next Steps -- [Features Guide](../user-guide/features.md) - Deep dive into all indicators -- [Labeling Guide](../user-guide/labeling.md) - Triple-barrier and other labeling methods +- [Features Guide](../user-guide/features.md) - 120 indicators across 11 categories +- [Labeling Guide](../user-guide/labeling.md) - 7 labeling methods for supervised learning +- [Alternative Bars](../user-guide/bars.md) - Information-driven bar sampling +- [Feature Discovery](../user-guide/discovery.md) - Registry, catalog, and search API +- [Fractional Differencing](../user-guide/fractional-differencing.md) - Memory-preserving stationarity +- [Dataset Builder](../user-guide/dataset-builder.md) - Leakage-safe train/test preparation - [API Reference](../api/index.md) - Complete API documentation diff --git a/docs/index.md b/docs/index.md index adbc97d..5772528 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,6 +53,17 @@ pip install ml4t-engineer - [Installation Guide](getting-started/installation.md) - Detailed setup instructions - [Quickstart](getting-started/quickstart.md) - Get running in 5 minutes + +### User Guides + +- [Features](user-guide/features.md) - 120 technical indicators across 11 categories +- [Labeling](user-guide/labeling.md) - 7 labeling methods for supervised learning +- [Alternative Bars](user-guide/bars.md) - Information-driven bar sampling +- [Feature Discovery](user-guide/discovery.md) - Registry, catalog, and search +- [Fractional Differencing](user-guide/fractional-differencing.md) - Memory-preserving stationarity +- [ML-Readiness](user-guide/ml-readiness.md) - Normalized features and preprocessing +- [Preprocessing](user-guide/preprocessing.md) - Scalers and leakage prevention +- [Dataset Builder](user-guide/dataset-builder.md) - Leakage-safe train/test preparation - [API Reference](api/index.md) - Complete API documentation ## Part of the ML4T Library Suite diff --git a/docs/user-guide/bars.md b/docs/user-guide/bars.md index b56215b..52b0615 100644 --- a/docs/user-guide/bars.md +++ b/docs/user-guide/bars.md @@ -2,6 +2,8 @@ Transform tick data into information-driven bars instead of time-based bars. +> **Book**: *ML for Trading, 3rd ed.* — Ch3 `08_itch_bar_sampling.py` constructs tick, volume, and dollar bars from ITCH trade data. `10_itch_information_bars.py` builds imbalance bars with threshold analysis. `13_databento_bar_sampling.py` demonstrates bar sampling on Databento data. + ## Why Alternative Bars? Time bars (1min, 1h, daily) have problems: diff --git a/docs/user-guide/dataset-builder.md b/docs/user-guide/dataset-builder.md new file mode 100644 index 0000000..5c349cb --- /dev/null +++ b/docs/user-guide/dataset-builder.md @@ -0,0 +1,203 @@ +# Dataset Builder + +`MLDatasetBuilder` provides leakage-safe dataset preparation for ML training. It handles train/test splitting, automatic scaling (fit on train only), and cross-validation integration with proper fold-level preprocessing. + +> **Book**: *ML for Trading, 3rd ed.* — Ch7 `10_ml4t_library_ecosystem.py` demonstrates `MLDatasetBuilder` with triple-barrier labels: features + labels in, scaled train/test split out. Ch7 `02_preprocessing_pipeline.py` covers the underlying preprocessing concepts. + +## Basic Usage + +```python +from ml4t.engineer import create_dataset_builder + +builder = create_dataset_builder( + features=features_df, # pl.DataFrame of feature columns + labels=labels_series, # pl.Series of target labels + dates=dates_series, # Optional: pl.Series of timestamps + scaler="standard", # "standard", "minmax", "robust", or None +) +``` + +### Train/Test Split + +```python +X_train, X_test, y_train, y_test = builder.train_test_split( + train_size=0.8, + shuffle=False, # Keep False for time series! + random_state=None, +) +``` + +When a scaler is set, `train_test_split` automatically: + +1. Fits the scaler on `X_train` only +2. Transforms both `X_train` and `X_test` using training statistics +3. Returns scaled DataFrames + +This prevents information leakage by construction. + +### Dataset Info + +```python +info = builder.info +# DatasetInfo( +# n_samples=2516, +# n_features=45, +# feature_names=["rsi", "macd", "atr", ...], +# label_name="label", +# has_dates=True, +# ) +``` + +## Scaler Configuration + +```python +from ml4t.engineer.preprocessing import StandardScaler, MinMaxScaler, RobustScaler + +# Via string shorthand +builder = create_dataset_builder(features, labels, scaler="standard") +builder = create_dataset_builder(features, labels, scaler="robust") +builder = create_dataset_builder(features, labels, scaler="minmax") +builder = create_dataset_builder(features, labels, scaler=None) # No scaling + +# Via scaler instance (custom parameters) +builder = create_dataset_builder( + features, labels, + scaler=RobustScaler(quantile_range=(10.0, 90.0)), +) + +# Change scaler after construction +builder.set_scaler(MinMaxScaler(feature_range=(0, 1))) +builder.set_scaler(None) # Disable scaling +``` + +## Cross-Validation Integration + +`MLDatasetBuilder` integrates with any splitter that follows the `SplitterProtocol` (compatible with ml4t-diagnostic's `WalkForwardCV` and `CombinatorialCV`). + +```python +for fold in builder.split(cv=splitter): + # Each fold has properly scaled train/test data + fold.X_train # pl.DataFrame (scaled with train-only stats) + fold.X_test # pl.DataFrame (scaled with train stats) + fold.y_train # pl.Series + fold.y_test # pl.Series + fold.fold_number # int + fold.scaler # Fitted BaseScaler (or None) + fold.train_indices # np.ndarray + fold.test_indices # np.ndarray + + # Convert to numpy for sklearn/lightgbm + X_np, X_test_np, y_np, y_test_np = fold.to_numpy() + + # Train your model + model.fit(X_np, y_np) + preds = model.predict(X_test_np) +``` + +Each fold gets its own scaler instance, fitted independently on that fold's training data. This is the correct behavior for time-series cross-validation where the training window shifts. + +### FoldResult + +The `FoldResult` dataclass returned by each iteration: + +```python +@dataclass +class FoldResult: + X_train: pl.DataFrame + X_test: pl.DataFrame + y_train: pl.Series + y_test: pl.Series + train_indices: NDArray[np.intp] + test_indices: NDArray[np.intp] + fold_number: int + scaler: BaseScaler | None = None + + def to_numpy(self) -> tuple[NDArray, NDArray, NDArray, NDArray] +``` + +## Percentile Computation + +For creating training-only thresholds (e.g., percentile-based labels): + +```python +# Feature percentiles (training data only) +cutoffs = builder.get_feature_percentiles( + train_idx=train_indices, + quantiles=[0.1, 0.25, 0.5, 0.75, 0.9], +) + +# Label percentiles (for discretizing continuous targets) +label_cutoffs = builder.compute_label_percentiles( + train_idx=train_indices, + n_quantiles=5, +) +``` + +These methods ensure percentile thresholds are computed from training data only, preventing look-ahead bias. + +## Format Conversion + +```python +# To numpy (raw, no scaling applied) +X_np, y_np = builder.to_numpy() + +# To pandas +X_pd, y_pd = builder.to_pandas() +``` + +## Factory Function + +The `create_dataset_builder` factory provides convenient scaler configuration: + +```python +from ml4t.engineer import create_dataset_builder + +builder = create_dataset_builder( + features=features_df, + labels=labels_series, + dates=dates_series, # Optional timestamps + scaler="standard", # str, BaseScaler, PreprocessingConfig, or None +) +``` + +The `scaler` parameter accepts: + +| Value | Effect | +|-------|--------| +| `"standard"` | StandardScaler with defaults | +| `"minmax"` | MinMaxScaler with (0, 1) range | +| `"robust"` | RobustScaler with IQR | +| `None` | No scaling | +| `BaseScaler` instance | Custom scaler with your parameters | +| `PreprocessingConfig` | Config object that creates the scaler | + +## End-to-End Example + +```python +import polars as pl +from ml4t.engineer import compute_features, create_dataset_builder +from ml4t.engineer.config import LabelingConfig +from ml4t.engineer.labeling import triple_barrier_labels + +# 1. Compute features +df = pl.read_parquet("spy_daily.parquet") +features_df = compute_features(df, ["rsi", "macd", "atr", "bollinger_bands"]) + +# 2. Create labels +config = LabelingConfig.triple_barrier( + upper_barrier=0.02, lower_barrier=0.01, max_holding_period=20, +) +labeled = triple_barrier_labels(features_df, config=config) + +# 3. Build dataset +feature_cols = [c for c in features_df.columns if c not in df.columns] +builder = create_dataset_builder( + features=labeled.select(feature_cols), + labels=labeled["label"], + dates=labeled["timestamp"], + scaler="robust", +) + +# 4. Train/test split with automatic scaling +X_train, X_test, y_train, y_test = builder.train_test_split(train_size=0.8) +``` diff --git a/docs/user-guide/discovery.md b/docs/user-guide/discovery.md new file mode 100644 index 0000000..100bd9c --- /dev/null +++ b/docs/user-guide/discovery.md @@ -0,0 +1,162 @@ +# Feature Discovery + +ML4T Engineer provides two complementary discovery APIs: the **Feature Registry** for programmatic metadata access, and the **Feature Catalog** for interactive exploration with filtering and search. + +## Feature Registry + +The registry is the metadata backbone — every feature registers its name, category, parameters, input requirements, and validation status. + +```python +from ml4t.engineer.core.registry import get_registry + +registry = get_registry() +``` + +### List Features + +```python +# All 120 features (sorted alphabetically) +all_features = registry.list_all() + +# By category +momentum = registry.list_by_category("momentum") # 31 features + +# By property +normalized = registry.list_normalized() # 37 ML-ready features +ta_lib = registry.list_ta_lib_compatible() # 59 validated features +``` + +### Inspect Metadata + +```python +meta = registry.get("rsi") + +meta.name # "rsi" +meta.category # "momentum" +meta.description # "Relative Strength Index" +meta.formula # "RSI = 100 - 100/(1 + RS), RS = AvgGain/AvgLoss" +meta.parameters # {"period": 14} +meta.input_type # "OHLCV" +meta.output_type # "indicator" +meta.normalized # True +meta.value_range # (0, 100) +meta.ta_lib_compatible # True +meta.dependencies # [] +meta.references # ["Wilder, 1978"] +meta.tags # ["oscillator", "overbought", "oversold"] +``` + +### Get Dependencies + +Some features depend on others (e.g., `stochrsi` depends on `rsi`): + +```python +deps = registry.get_dependencies("stochrsi") # ["rsi"] +``` + +`compute_features` resolves these automatically via topological sort. + +## Feature Catalog + +The catalog wraps the registry with higher-level filtering and full-text search: + +```python +from ml4t.engineer import feature_catalog +``` + +### Filtered Listing + +```python +# Single filter +feature_catalog.list(category="volatility") + +# Multiple filters (AND logic) +feature_catalog.list( + category="momentum", + normalized=True, + ta_lib_compatible=True, +) + +# All available filters +feature_catalog.list( + category=None, # str: filter by category + normalized=None, # bool: ML-ready features only + ta_lib_compatible=None, # bool: TA-Lib validated only + tags=None, # list[str]: features matching any tag + input_type=None, # str: "OHLCV", "close", "returns", etc. + output_type=None, # str: "indicator", "bands", etc. + has_dependencies=None, # bool: features with/without dependencies + limit=None, # int: max results +) +``` + +### Full-Text Search + +Search across feature names, descriptions, tags, and formulas: + +```python +results = feature_catalog.search("volatility estimator") +# [("parkinson_volatility", 0.65), ("garman_klass_volatility", 0.45), ("rogers_satchell_volatility", 0.45), ...] + +results = feature_catalog.search("trend strength") +# [("trend_intensity_index", 0.92), ("adx", 0.78), ...] + +results = feature_catalog.search("spread") +# [("realized_spread", 1.30), ("roll_spread_estimator", 1.30)] +``` + +Returns a list of `(feature_name, relevance_score)` tuples, sorted by relevance. + +### Describe + +Get a dict summary of any feature: + +```python +info = feature_catalog.describe("yang_zhang_volatility") +# { +# "name": "yang_zhang_volatility", +# "category": "volatility", +# "description": "Yang-Zhang Volatility - combines overnight and intraday volatility", +# "parameters": {}, +# "normalized": False, +# "ta_lib_compatible": False, +# "input_type": "close", +# "value_range": None, +# "dependencies": [], +# "tags": [], +# } +``` + +### Browse Categories and Tags + +```python +feature_catalog.categories() +# ['cross_asset', 'math', 'microstructure', 'ml', 'momentum', +# 'price_transform', 'regime', 'risk', 'statistics', 'trend', +# 'volatility', 'volume'] + +feature_catalog.tags() +# ['efficient', 'illiquidity', 'ma', 'microstructure', 'normalized', +# 'ohlc', 'oscillator', 'overbought', 'oversold', 'spread', ...] +``` + +## Metadata Fields Reference + +| Field | Type | Description | +|-------|------|-------------| +| `name` | `str` | Unique feature identifier | +| `category` | `str` | Feature category | +| `description` | `str` | One-line description | +| `formula` | `str` | Mathematical formula | +| `parameters` | `dict[str, Any]` | Default parameters | +| `input_type` | `str` | Required input columns (`"OHLCV"`, `"close"`, etc.) | +| `output_type` | `str` | Output type (`"indicator"`, `"bands"`, etc.) | +| `normalized` | `bool \| None` | Whether output is bounded | +| `value_range` | `tuple[float, float] \| None` | Output range if normalized | +| `ta_lib_compatible` | `bool` | Validated against TA-Lib at 1e-6 | +| `dependencies` | `list[str]` | Other features this depends on | +| `references` | `list[str]` | Academic references | +| `tags` | `list[str]` | Searchable tags | +| `lookback` | `Callable` | Function returning minimum lookback period | + +> **Book**: *ML for Trading, 3rd ed.* — Ch7 `10_ml4t_library_ecosystem.py` explores the registry metadata for RSI, ATR, and Garman-Klass, then demonstrates `feature_catalog.search()` and filtered listing. diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index 9ab05a5..7a2ebf2 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -1,85 +1,390 @@ # Technical Indicators -ML4T Engineer provides 120 technical indicators across 11 categories. - -## Momentum Indicators (31) - -| Name | Description | TA-Lib Compatible | -|------|-------------|-------------------| -| `rsi` | Relative Strength Index | Yes | -| `macd` | Moving Average Convergence/Divergence | Yes | -| `stoch` | Stochastic Oscillator | Yes | -| `cci` | Commodity Channel Index | Yes | -| `willr` | Williams %R | Yes | -| `adx` | Average Directional Index | Yes | -| `mfi` | Money Flow Index | Yes | -| `roc` | Rate of Change | Yes | -| `mom` | Momentum | Yes | -| `trix` | Triple Exponential Average | Yes | - -## Trend Indicators (10) - -| Name | Description | TA-Lib Compatible | -|------|-------------|-------------------| -| `sma` | Simple Moving Average | Yes | -| `ema` | Exponential Moving Average | Yes | -| `wma` | Weighted Moving Average | Yes | -| `dema` | Double EMA | Yes | -| `tema` | Triple EMA | Yes | -| `kama` | Kaufman Adaptive MA | Yes | -| `t3` | Triple Exponential T3 | Yes | - -## Volatility Indicators (15) +ML4T Engineer provides 120 technical indicators across 11 categories, built on Polars with Numba JIT for performance-critical kernels. 59 indicators are validated against TA-Lib at 1e-6 tolerance. -| Name | Description | -|------|-------------| -| `atr` | Average True Range (TA-Lib) | -| `natr` | Normalized ATR (TA-Lib) | -| `bollinger_bands` | Bollinger Bands (TA-Lib) | -| `yang_zhang` | Yang-Zhang volatility (most efficient) | -| `parkinson` | Parkinson high-low volatility | -| `garman_klass` | Garman-Klass OHLC volatility | -| `rogers_satchell` | Rogers-Satchell drift-adjusted | -| `realized` | Realized volatility | -| `ewma` | EWMA volatility | -| `garch` | GARCH(1,1) forecast | - -## Microstructure (15) +## Overview -| Name | Description | -|------|-------------| -| `kyle_lambda` | Kyle's Lambda (price impact) | -| `amihud_illiquidity` | Amihud illiquidity ratio | -| `vpin` | Volume-Synchronized PIN | -| `roll_spread` | Roll implied spread | -| `corwin_schultz` | Corwin-Schultz high-low spread | -| `hasbrouck_lambda` | Hasbrouck's Lambda | +| Category | Count | TA-Lib | Key Indicators | +|----------|-------|--------|----------------| +| Momentum | 31 | 19 | RSI, MACD, Stochastic, CCI, ADX, MFI | +| Volatility | 15 | 4 | ATR, Bollinger, Yang-Zhang, Parkinson, GARCH | +| Microstructure | 15 | 0 | Kyle Lambda, Amihud, Roll Spread, Realized Spread | +| Trend | 10 | 9 | SMA, EMA, WMA, DEMA, TEMA, KAMA | +| ML Features | 14 | 0 | Lag, Entropy, Fourier, Cyclical Encode | +| Statistics | 14 | 7 | STDDEV, Linear Regression, TSF, Variance Ratio | +| Risk | 6 | 0 | Max Drawdown, Downside Deviation, Sharpe, Sortino | +| Price Transform | 5 | 5 | Typical Price, Weighted Close, Average Price | +| Regime | 4 | 0 | Hurst Exponent, Choppiness, Fractal Efficiency | +| Volume | 3 | 3 | OBV, AD, ADOSC | +| Math | 3 | 3 | Maximum, Minimum, Summation | +| Cross-Asset | 10 | 0 | Beta, Correlation, Cointegration (standalone functions) | -## ML Features (14) +> **Book**: *ML for Trading, 3rd ed.* — Ch8 notebooks (`01_price_volume_features.py` through `04_fundamentals_macro_calendar.py`) build features manually to explain the economics. Case studies (ETFs, US Equities Panel, CME Futures) then use `compute_features()` in production pipelines. -| Name | Description | -|------|-------------| -| `fractional_diff` | Fractionally differenced series | -| `lag` | Lagged values | -| `rolling_stats` | Rolling statistics | -| `entropy` | Shannon entropy | -| `hurst` | Hurst exponent | -| `autocorr` | Autocorrelation | +## Computation API -## Usage Examples +`compute_features()` accepts three input formats: ```python from ml4t.engineer import compute_features -# Single indicator -result = compute_features(df, ["rsi"]) +# 1. List of names (default parameters) +result = compute_features(df, ["rsi", "macd", "atr"]) -# Multiple indicators -result = compute_features(df, ["rsi", "macd", "bollinger_bands"]) - -# Custom parameters +# 2. List of dicts (custom parameters) result = compute_features(df, [ {"name": "rsi", "params": {"period": 20}}, {"name": "sma", "params": {"period": 50}}, + {"name": "bollinger_bands", "params": {"period": 20, "std_dev": 2.5}}, ]) + +# 3. YAML config file (production pipelines) +result = compute_features(df, "features.yaml") +``` + +Features are computed in dependency order (topological sort). Circular dependencies raise `ValueError`. The return type matches the input: `DataFrame` in, `DataFrame` out; `LazyFrame` in, `LazyFrame` out. + +> **Book**: Ch7 `10_ml4t_library_ecosystem.py` demonstrates all three input formats on SPY data, including a comparison between library and manual RSI implementations. + +## Category Reference + +### Momentum (31 indicators) + +Price momentum and oscillator indicators. Most produce bounded (normalized) output suitable for direct ML use. + +| Name | Description | TA-Lib | Normalized | Default Period | +|------|-------------|--------|------------|----------------| +| `rsi` | Relative Strength Index | Yes | 0-100 | 14 | +| `macd` | Moving Average Convergence/Divergence | Yes | No | 12/26/9 | +| `stochastic` | Stochastic Oscillator (%K, %D) | No | 0-100 | 14/3/3 | +| `stochf` | Fast Stochastic | Yes | 0-100 | 5/3 | +| `stochrsi` | Stochastic RSI | Yes | 0-100 | 14 | +| `cci` | Commodity Channel Index | Yes | ~-200 to 200 | 14 | +| `willr` | Williams %R | Yes | -100 to 0 | 14 | +| `adx` | Average Directional Index | Yes | 0-100 | 14 | +| `adxr` | ADX Rating | Yes | 0-100 | 14 | +| `dx` | Directional Movement Index | Yes | 0-100 | 14 | +| `plus_di` | Plus Directional Indicator | Yes | 0-100 | 14 | +| `minus_di` | Minus Directional Indicator | Yes | 0-100 | 14 | +| `mfi` | Money Flow Index | Yes | 0-100 | 14 | +| `roc` | Rate of Change | Yes | No | 10 | +| `rocp` | Rate of Change (%) | Yes | No | 10 | +| `mom` | Momentum | Yes | No | 10 | +| `trix` | Triple Exponential Average | Yes | No | 30 | +| `cmo` | Chande Momentum Oscillator | Yes | -100 to 100 | 14 | +| `ultosc` | Ultimate Oscillator | Yes | 0-100 | 7/14/28 | +| `bop` | Balance of Power | Yes | -1 to 1 | — | +| `imi` | Intraday Momentum Index | No | 0-100 | 14 | +| `aroon` | Aroon (up/down) | Yes | 0-100 | 14 | +| `aroonosc` | Aroon Oscillator | Yes | -100 to 100 | 14 | +| `apo` | Absolute Price Oscillator | Yes | No | 12/26 | +| `ppo` | Percentage Price Oscillator | Yes | No | 12/26 | +| `sar` | Parabolic SAR | Yes | No | 0.02/0.2 | + +> **Book**: Ch8 `01_price_volume_features.py` constructs momentum indicators on ETF data, explaining the economic rationale for each. ETFs and US Equities Panel case studies use these in `03_features.py`. + +### Trend (10 indicators) + +Moving averages that produce price-scale outputs. Require preprocessing for ML models. + +| Name | Description | TA-Lib | Default Period | +|------|-------------|--------|----------------| +| `sma` | Simple Moving Average | Yes | 20 | +| `ema` | Exponential Moving Average | Yes | 20 | +| `wma` | Weighted Moving Average | Yes | 20 | +| `dema` | Double Exponential MA | Yes | 20 | +| `tema` | Triple Exponential MA | Yes | 20 | +| `t3` | Triple Exponential T3 | Yes | 5 | +| `kama` | Kaufman Adaptive MA | Yes | 30 | +| `trima` | Triangular MA | Yes | 20 | +| `midpoint` | Midpoint over period | Yes | 14 | +| `donchian_channels` | Donchian Channels (highest high/lowest low) | No | 20 | + +### Volatility (15 indicators) + +Volatility estimators ranging from simple (ATR) to advanced (GARCH). Includes range-based estimators that are more efficient than close-to-close. + +| Name | Description | TA-Lib | Normalized | +|------|-------------|--------|------------| +| `atr` | Average True Range | Yes | No | +| `natr` | Normalized ATR (% of price) | Yes | 0-100 | +| `trange` | True Range | Yes | No | +| `bollinger_bands` | Bollinger Bands (upper/middle/lower) | Yes | No | +| `yang_zhang_volatility` | Yang-Zhang (overnight + intraday) | No | No | +| `parkinson_volatility` | Parkinson range-based | No | No | +| `garman_klass_volatility` | Garman-Klass OHLC-based | No | No | +| `rogers_satchell_volatility` | Rogers-Satchell drift-independent | No | No | +| `realized_volatility` | Standard deviation of returns | No | No | +| `ewma_volatility` | EWMA of variance | No | No | +| `garch_forecast` | GARCH(1,1) conditional volatility | No | No | +| `conditional_volatility_ratio` | Up-market vs down-market vol ratio | No | No | +| `volatility_percentile_rank` | Current vol vs historical distribution | No | 0-100 | +| `volatility_of_volatility` | Second-order volatility measure | No | No | +| `volatility_regime_probability` | High/low vol regime probability | No | No | + +**Efficiency ranking**: Yang-Zhang > Garman-Klass ~ Rogers-Satchell > Parkinson > Close-to-Close. See Molnar (2012) for theoretical efficiency ratios. + +> **Book**: Ch9 `08_garch_volatility.py` and `09_har_rough_volatility.py` compare volatility estimators on real data. Ch8 `01_price_volume_features.py` covers range-based estimators with efficiency analysis. + +### Microstructure (15 indicators) + +Market microstructure features from De Prado (2018) and empirical market microstructure literature. + +| Name | Description | +|------|-------------| +| `kyle_lambda` | Kyle's Lambda (price impact coefficient) | +| `amihud_illiquidity` | Amihud illiquidity ratio | +| `roll_spread_estimator` | Roll implied bid-ask spread | +| `realized_spread` | Realized spread | +| `effective_tick_rule` | Effective tick rule classification | +| `order_flow_imbalance` | Order flow imbalance | +| `price_impact_ratio` | Price impact ratio | +| `volume_weighted_price_momentum` | Volume-weighted price momentum | +| `bid_ask_imbalance` | Bid-ask imbalance (normalized -1 to 1) | +| `book_depth_ratio` | Book depth ratio (normalized 0 to 1) | +| `quote_stuffing_indicator` | Quote stuffing detection | +| `trade_intensity` | Trade intensity | +| `volume_at_price_ratio` | Volume at price ratio | +| `volume_synchronicity` | Volume synchronicity | +| `weighted_mid_price` | Weighted mid price | + +> **Book**: Ch8 `02_microstructure_features.py` builds microstructure features from tick and minute data. The NASDAQ-100 Microstructure case study (`03_features.py`) implements Kyle's Lambda, Amihud, and VPIN manually for pedagogical purposes — the ml4t-engineer implementations are production-ready equivalents. + +### ML Features (14 indicators) + +Features designed specifically for machine learning pipelines. + +| Name | Description | Normalized | +|------|-------------|------------| +| `create_lag_features` | Multiple lag columns at once | No | +| `cyclical_encode` | Cyclical time encoding (sin/cos) | No | +| `fourier_features` | Fourier transform features | No | +| `rolling_entropy` | Shannon entropy | 0-10 | +| `rolling_entropy_lz` | Lempel-Ziv entropy | 0-10 | +| `rolling_entropy_plugin` | Plugin entropy estimator | 0-10 | +| `percentile_rank_features` | Rank-based normalization | 0-100 | +| `interaction_features` | Feature interaction terms | No | +| `multi_horizon_returns` | Returns at multiple horizons | No | +| `directional_targets` | Directional movement targets | No | +| `volatility_adjusted_returns` | Returns scaled by volatility | No | +| `regime_conditional_features` | Regime-conditional transforms | No | +| `time_decay_weights` | Exponential time decay | No | +| `ffdiff` | Fractional differencing | No | + +> **Book**: Ch8 `04_fundamentals_macro_calendar.py` covers feature construction patterns including lag features and calendar encodings. + +### Risk (6 indicators) + +Risk and risk-adjusted return metrics. + +| Name | Description | Normalized | +|------|-------------|------------| +| `maximum_drawdown` | Maximum drawdown | No | +| `downside_deviation` | Downside volatility | 0-2 | +| `tail_ratio` | Right tail / left tail ratio | 0-10 | +| `higher_moments` | Skewness and kurtosis | No | +| `risk_adjusted_returns` | Sharpe, Sortino, Calmar, Omega | No | +| `ulcer_index` | Ulcer Index (drawdown-based risk) | No | + +### Cross-Asset (10 functions) + +Multi-asset relationship features. These are standalone functions in `ml4t.engineer.features.cross_asset` rather than registry entries, since they require two or more price series as input. + +| Function | Description | +|----------|-------------| +| `rolling_correlation` | Rolling Pearson correlation | +| `beta_to_market` | Rolling beta vs market index | +| `correlation_regime_indicator` | Low/medium/high correlation regimes | +| `lead_lag_correlation` | Lead-lag cross-correlation | +| `multi_asset_dispersion` | Cross-sectional return dispersion | +| `correlation_matrix_features` | Mean/min/max of correlation matrix | +| `relative_strength_index_spread` | RSI spread between two assets | +| `volatility_ratio` | Volatility ratio between assets | +| `co_integration_score` | Rolling cointegration score | +| `cross_asset_momentum` | Rank-based cross-asset momentum | + +These are called directly (not via `compute_features`) since they require multi-asset DataFrames. + +> **Book**: Ch8 `03_structural_cross_instrument_features.py` constructs cross-asset features. Ch9 `14_panel_features.py` applies cross-sectional features to equity panels. + +### Regime (4 indicators) + +Market regime detection features. All produce bounded outputs suitable for direct ML use. + +| Name | Description | Range | +|------|-------------|-------| +| `hurst_exponent` | Hurst exponent (R/S analysis) | 0-1 | +| `choppiness_index` | Market choppiness | 0-100 | +| `fractal_efficiency` | Price path efficiency | 0-1 | +| `trend_intensity_index` | Trend strength | 0-100 | + +> **Book**: Ch9 `11_hmm_regimes.py` and `13_regime_as_feature.py` apply regime detection to equity indices. + +### Statistics (14 indicators) + +Statistical features including TA-Lib standard and rolling distribution metrics. + +| Name | Description | TA-Lib | Normalized | +|------|-------------|--------|------------| +| `stddev` | Standard Deviation | Yes | No | +| `var` | Variance | Yes | No | +| `avgdev` | Average Deviation | No | No | +| `linearreg` | Linear Regression Value | Yes | No | +| `linearreg_slope` | Linear Regression Slope | Yes | No | +| `linearreg_angle` | Linear Regression Angle | Yes | No | +| `linearreg_intercept` | Linear Regression Intercept | Yes | No | +| `tsf` | Time Series Forecast | Yes | No | +| `coefficient_of_variation` | Rolling coefficient of variation | No | 0-10 | +| `variance_ratio` | Variance ratio test | No | 0-5 | +| `rolling_cv_zscore` | Cross-validated z-score | No | -10 to 10 | +| `rolling_drift` | Rolling drift estimate | No | -10 to 10 | +| `rolling_kl_divergence` | KL divergence vs reference | No | 0-10 | +| `rolling_wasserstein` | Wasserstein distance | No | No | + +### Price Transform (5 indicators) + +| Name | Description | TA-Lib | +|------|-------------|--------| +| `avgprice` | Average Price (O+H+L+C)/4 | Yes | +| `typprice` | Typical Price (H+L+C)/3 | Yes | +| `medprice` | Median Price (H+L)/2 | Yes | +| `wclprice` | Weighted Close (H+L+2C)/4 | Yes | +| `midprice` | Midpoint Price (H+L)/2 | Yes | + +### Volume (3 indicators) + +| Name | Description | TA-Lib | +|------|-------------|--------| +| `obv` | On Balance Volume | Yes | +| `ad` | Accumulation/Distribution | Yes | +| `adosc` | A/D Oscillator | Yes | + +> **Book**: ETFs case study `03_features.py` uses volume features in a multi-asset pipeline alongside momentum and volatility. + +### Math (3 indicators) + +O(n) rolling operations using monotonic deque. + +| Name | Description | TA-Lib | +|------|-------------|--------| +| `maximum` | Rolling maximum | Yes | +| `minimum` | Rolling minimum | Yes | +| `summation` | Rolling sum | Yes | + +### Fractional Differencing (4 functions) + +See the dedicated [Fractional Differencing guide](fractional-differencing.md) for the full workflow. + +## Feature Discovery + +The `FeatureCatalog` provides filtering and full-text search over all 120 features: + +```python +from ml4t.engineer import feature_catalog + +# List by category +momentum = feature_catalog.list(category="momentum") + +# Filter by multiple criteria +ml_ready = feature_catalog.list(normalized=True, ta_lib_compatible=True) + +# Full-text search +results = feature_catalog.search("volatility estimator") +# Returns: [("parkinson_volatility", 0.65), ("garman_klass_volatility", 0.45), ...] + +# Detailed feature info +info = feature_catalog.describe("yang_zhang_volatility") +# {'name': 'yang_zhang_volatility', 'category': 'volatility', ...} + +# List all categories +print(feature_catalog.categories()) +# ['cross_asset', 'math', 'microstructure', 'ml', 'momentum', ...] + +# List all tags +print(feature_catalog.tags()) ``` + +See the dedicated [Feature Discovery guide](discovery.md) for complete examples. + +> **Book**: Ch7 `10_ml4t_library_ecosystem.py` explores the registry metadata for RSI, ATR, and Garman-Klass, then demonstrates `feature_catalog.search()` and filtered listing. + +## YAML Configuration + +For reproducible feature pipelines, store configurations in YAML: + +```yaml +# features.yaml +features: + - name: rsi + params: + period: 14 + + - name: macd + params: + fast: 12 + slow: 26 + signal: 9 + + - name: bollinger_bands + params: + period: 20 + std_dev: 2.0 + + - name: yang_zhang_volatility +``` + +Load with `compute_features(df, "features.yaml")`. The YAML format supports version comments and parameter documentation inline. + +## Input Requirements + +### OHLCV DataFrame + +Most features expect a DataFrame with standardized column names (lowercase): + +| Column | Type | Required By | +|--------|------|-------------| +| `open` | float | OHLCV, OHLC features | +| `high` | float | OHLCV, OHLC, HLC, HL features | +| `low` | float | OHLCV, OHLC, HLC, HL features | +| `close` | float | All features | +| `volume` | float | OHLCV, volume features | +| `returns` | float | Return-based features (auto-computed if missing) | + +Features declare their `input_type` metadata (e.g., `"OHLCV"`, `"close"`, `"returns"`), and `compute_features` validates that required columns are present. + +### Missing Columns + +If a feature requires a column that's missing, `compute_features` raises a clear error: + +``` +ValueError: Feature 'mfi' requires column 'volume' (input_type='OHLCV') but it was not found. +``` + +## Custom Parameters + +Override default parameters per feature: + +```python +# Check defaults +from ml4t.engineer.core.registry import get_registry +meta = get_registry().get("rsi") +print(meta.parameters) # {'period': 14} + +# Override +result = compute_features(df, [{"name": "rsi", "params": {"period": 20}}]) +``` + +Invalid parameters raise `ValueError` with the valid parameter names. + +## Performance + +- **Polars-native**: All computations use Polars expressions for automatic parallelism +- **Numba JIT**: Numerical kernels (volatility estimators, microstructure) are Numba-accelerated +- **Throughput**: ~480K indicator calculations/second, 11M rows/second streaming +- **TA-Lib parity**: RSI computed at ~1x TA-Lib speed via Polars native implementation +- **Dependency ordering**: `compute_features` resolves feature dependencies via topological sort + +## References + +- Lopez de Prado, M. (2018). *Advances in Financial Machine Learning*. Wiley. +- Molnar, P. (2012). Properties of range-based volatility estimators. *International Review of Financial Analysis*. diff --git a/docs/user-guide/fractional-differencing.md b/docs/user-guide/fractional-differencing.md new file mode 100644 index 0000000..de1840a --- /dev/null +++ b/docs/user-guide/fractional-differencing.md @@ -0,0 +1,188 @@ +# Fractional Differencing + +Fractional differencing (FFD) produces stationary time series while preserving long-range memory — the key insight from De Prado (2018, Chapter 5). Standard first-differencing (d=1) achieves stationarity but destroys predictive signal; fractional differencing finds the minimum d that passes stationarity tests. + +## The Memory-Stationarity Tradeoff + +| Differencing Degree | Stationarity | Memory Preserved | ML Utility | +|---------------------|-------------|------------------|------------| +| d = 0 (original) | Non-stationary | 100% | Poor (unit root) | +| 0 < d < 0.5 | May be stationary | High | Optimal zone | +| d = 0.5 | Borderline | Moderate | Acceptable | +| d = 1 (first diff) | Stationary | ~0% | Poor (signal lost) | + +The goal: find the smallest d where the ADF test rejects the null hypothesis of a unit root (p-value < 0.05). + +## Core Functions + +### `ffdiff` — Apply Fractional Differencing + +```python +from ml4t.engineer.features.fdiff import ffdiff + +# As a Polars expression (chainable) +result = df.with_columns( + ffdiff("close", d=0.4, threshold=1e-5).alias("close_ffd") +) + +# Or on a Series directly +ffd_series = ffdiff(df["close"], d=0.4) +``` + +**Parameters**: + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `close` | `pl.Series \| pl.Expr \| str` | required | Price series or column name | +| `d` | `float` | required | Differencing degree (0 to 2) | +| `threshold` | `float` | `1e-5` | Minimum weight magnitude (truncation) | + +**How it works**: FFD applies a weighted sum of lagged values where weights are derived from the fractional binomial expansion. Weights decay geometrically, and the `threshold` parameter truncates negligibly small weights for efficiency. Weights are cached via `@lru_cache` and the inner loop is Numba-accelerated. + +### `find_optimal_d` — Find Minimum Stationary d + +```python +from ml4t.engineer.features.fdiff import find_optimal_d + +result = find_optimal_d( + close=df["close"], # Must be a pl.Series (not Expr) + d_range=(0.0, 1.0), # Search range + step=0.01, # Step size (finer = slower) + adf_pvalue_threshold=0.05, # ADF significance level +) + +print(result) +# {"optimal_d": 0.35, "adf_pvalue": 0.023, "correlation": 0.97} +``` + +**Returns** a dict with: + +| Key | Description | +|-----|-------------| +| `optimal_d` | Smallest d passing ADF test | +| `adf_pvalue` | ADF p-value at optimal d | +| `correlation` | Correlation between original and FFD series | + +A high correlation (>0.90) means most of the predictive information is preserved. + +### `fdiff_diagnostics` — Full Diagnostic Report + +```python +from ml4t.engineer.features.fdiff import fdiff_diagnostics + +diag = fdiff_diagnostics( + close=df["close"], + d=0.4, + threshold=1e-5, +) + +print(diag) +# {"d": 0.4, "adf_statistic": -3.21, "adf_pvalue": 0.019, +# "correlation": 0.96, "n_weights": 127, "weight_sum": 0.998} +``` + +## Step-by-Step Workflow + +### 1. Find Optimal d + +```python +import polars as pl +from ml4t.engineer.features.fdiff import find_optimal_d, ffdiff + +# Load price data +df = pl.read_parquet("spy_daily.parquet") + +# Find minimum d for stationarity +result = find_optimal_d(df["close"], step=0.01) +optimal_d = result["optimal_d"] +print(f"Optimal d: {optimal_d}, ADF p-value: {result['adf_pvalue']:.4f}") +print(f"Correlation with original: {result['correlation']:.4f}") +``` + +### 2. Apply Fractional Differencing + +```python +# Apply FFD with optimal d +df = df.with_columns( + ffdiff("close", d=optimal_d).alias("close_ffd") +) +``` + +### 3. Validate with ADF Test + +```python +from statsmodels.tsa.stattools import adfuller + +adf_result = adfuller(df["close_ffd"].drop_nulls().to_numpy()) +print(f"ADF statistic: {adf_result[0]:.4f}") +print(f"p-value: {adf_result[1]:.6f}") +# Should be < 0.05 +``` + +### 4. Use as ML Feature + +```python +# FFD series is now stationary but retains memory +# Use alongside other features +from ml4t.engineer import compute_features + +features = compute_features(df, ["rsi", "macd", "atr"]) +features = features.with_columns( + ffdiff("close", d=optimal_d).alias("close_ffd") +) +``` + +## Via compute_features + +Fractional differencing is also available through the standard `compute_features` API: + +```python +result = compute_features(df, [ + {"name": "fractional_diff", "params": {"d": 0.4}}, +]) +``` + +Or find the optimal d and apply it: + +```python +result = compute_features(df, [ + {"name": "ffdiff_optimal", "params": {"adf_pvalue_threshold": 0.05}}, +]) +``` + +## Asset-Class Guidelines + +Typical optimal d values (these are starting points — always validate on your data): + +| Asset Class | Typical d Range | Notes | +|-------------|----------------|-------| +| Equity indices (SPY, QQQ) | 0.3 - 0.5 | Strong trend component | +| Individual stocks | 0.3 - 0.6 | Higher variance, may need larger d | +| Futures (ES, NQ) | 0.2 - 0.4 | Session structure affects ADF | +| FX pairs | 0.3 - 0.5 | Mean-reverting pairs may need lower d | +| Crypto | 0.4 - 0.7 | High volatility, regime-dependent | + +### Multi-Asset Application + +For multi-asset pipelines, compute optimal d per asset: + +```python +for symbol in ["SPY", "QQQ", "IWM"]: + asset_data = df.filter(pl.col("symbol") == symbol) + result = find_optimal_d(asset_data["close"]) + print(f"{symbol}: d={result['optimal_d']}, corr={result['correlation']:.3f}") +``` + +## Performance + +- Weights computed once and cached (`@lru_cache`) +- Inner loop Numba-accelerated +- Weight truncation via `threshold` limits computation window +- Typical: 383 lines of implementation for the full module + +> **Book**: *ML for Trading, 3rd ed.* — Ch9 `03_fractional_differencing.py` applies FFD to equity data with ADF test validation and memory-stationarity analysis. The ETFs and US Equities Panel case studies use FFD in their `04_temporal.py` notebooks as a standard feature preparation step. + +## References + +- Lopez de Prado, M. (2018). *Advances in Financial Machine Learning*. Wiley. Chapter 5: Fractionally Differentiated Features. +- Hosking, J.R.M. (1981). Fractional differencing. *Biometrika*, 68(1), 165-176. diff --git a/docs/user-guide/labeling.md b/docs/user-guide/labeling.md index f96631e..208f24f 100644 --- a/docs/user-guide/labeling.md +++ b/docs/user-guide/labeling.md @@ -1,57 +1,521 @@ # Labeling Methods -ML4T Engineer provides advanced labeling methods for supervised learning in finance. +ML4T Engineer provides 7 labeling methods for supervised learning in finance, implementing the full workflow from *Advances in Financial Machine Learning* (De Prado, 2018). -## Triple-Barrier Labeling +## Overview -The triple-barrier method from *Advances in Financial Machine Learning* creates labels based on which barrier is touched first: +| Method | Function | Use Case | +|--------|----------|----------| +| Triple-barrier | `triple_barrier_labels()` | Fixed profit/loss targets with time limit | +| ATR-based barriers | `atr_triple_barrier_labels()` | Volatility-adjusted targets | +| Rolling percentile | `rolling_percentile_binary_labels()` | Adaptive threshold from return distribution | +| Fixed time horizon | `fixed_time_horizon_labels()` | Simple forward returns | +| Trend scanning | `trend_scanning_labels()` | Optimal horizon via t-statistic | +| Meta-labeling | `meta_labels()` + `compute_bet_size()` | Bet sizing for primary model | +| Calendar-aware | `calendar_aware_labels()` | Session-break handling for futures | -- **Upper barrier**: Profit target (label = 1) -- **Lower barrier**: Stop loss (label = -1) -- **Vertical barrier**: Time limit (label = 0) +All methods return a Polars DataFrame with standardized output columns. Performance is ~50,000 labels/second via Numba-accelerated kernels. + +> **Book**: *ML for Trading, 3rd ed.* — Ch7 `03_label_methods.py` walks through all 7 methods on real ETF data with visualizations. All case study `02_labels.py` notebooks apply these methods in production pipelines. + +## Choosing a Method + +``` +Need fixed profit/loss targets? +├── Yes → Do barriers scale with volatility? +│ ├── Yes → atr_triple_barrier_labels() +│ └── No → triple_barrier_labels() +├── No → Need directional labels (long/short)? +│ ├── Yes → rolling_percentile_binary_labels() +│ └── No → Need optimal holding period? +│ ├── Yes → trend_scanning_labels() +│ └── No → fixed_time_horizon_labels() + +Have a primary model? → meta_labels() for bet sizing +Trading futures with session breaks? → calendar_aware_labels() +``` + +## LabelingConfig + +All barrier-based methods accept a `LabelingConfig` object created via factory methods. This provides serialization, validation, and a bridge to `DataContractConfig` for pipeline integration. + +### Factory Methods + +```python +from ml4t.engineer.config import LabelingConfig + +# Fixed barriers +config = LabelingConfig.triple_barrier( + upper_barrier=0.02, # 2% take profit + lower_barrier=0.01, # 1% stop loss + max_holding_period=20, # 20 bars (or "4h" for time-based) + side=1, # 1=long, -1=short, 0=symmetric + trailing_stop=False, # Enable trailing stop loss +) + +# ATR-based barriers +config = LabelingConfig.atr_barrier( + atr_tp_multiple=2.0, # 2x ATR take profit + atr_sl_multiple=1.0, # 1x ATR stop loss + atr_period=14, + max_holding_period=20, +) + +# Fixed horizon (simple forward returns) +config = LabelingConfig.fixed_horizon( + horizon=10, + return_method="returns", # "returns" | "log_returns" | "binary" + threshold=None, +) + +# Trend scanning +config = LabelingConfig.trend_scanning( + min_horizon=5, + max_horizon=20, + t_value_threshold=2.0, +) +``` + +### Serialization + +Store labeling configurations for experiment reproducibility: + +```python +# Save to YAML +config.to_yaml("labeling_config.yaml") + +# Reload +config = LabelingConfig.from_yaml("labeling_config.yaml") +``` + +## Triple-Barrier Labels + +The triple-barrier method from AFML Chapter 3 creates labels based on which of three barriers is touched first: upper (profit target), lower (stop loss), or vertical (time limit). ```python from ml4t.engineer.config import LabelingConfig from ml4t.engineer.labeling import triple_barrier_labels config = LabelingConfig.triple_barrier( - upper_barrier=0.02, # 2% profit target - lower_barrier=0.01, # 1% stop loss - max_holding_period=20, # 20 bar horizon + upper_barrier=0.02, # 2% profit target + lower_barrier=0.01, # 1% stop loss + max_holding_period=20, # 20 bar horizon + side=1, # Long-only signals ) -labels = triple_barrier_labels( - df, +result = triple_barrier_labels( + data=df, config=config, + price_col="close", + high_col="high", # For intrabar barrier touches + low_col="low", # For intrabar barrier touches + timestamp_col="timestamp", # Required for time-based max_holding + calculate_uniqueness=False, # Compute sample weights + uniqueness_weight_scheme="returns_uniqueness", ) ``` +### Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `data` | `pl.DataFrame` | required | OHLCV data | +| `config` | `LabelingConfig` | required | Barrier configuration | +| `price_col` | `str` | `"close"` | Price column for barrier calculations | +| `high_col` | `str \| None` | `None` | High column for intrabar touch detection | +| `low_col` | `str \| None` | `None` | Low column for intrabar touch detection | +| `timestamp_col` | `str \| None` | `None` | Required when `max_holding_period` is time-based | +| `calculate_uniqueness` | `bool` | `False` | Compute label uniqueness and sample weights | +| `uniqueness_weight_scheme` | `str` | `"returns_uniqueness"` | Weight scheme (see Sample Weighting) | + +### Output Columns + +| Column | Description | +|--------|-------------| +| `label` | +1 (upper hit), -1 (lower hit), 0 (vertical hit) | +| `label_time` | Timestamp when barrier was hit | +| `label_price` | Price at barrier touch | +| `label_return` | Return from entry to barrier | +| `label_bars` | Number of bars until barrier | +| `label_duration` | Time duration until barrier | +| `barrier_hit` | Which barrier: `"upper"`, `"lower"`, `"vertical"` | +| `label_uniqueness` | Average uniqueness (when `calculate_uniqueness=True`) | +| `sample_weight` | Sample weight (when `calculate_uniqueness=True`) | + +### The `side` Parameter + +Controls directional bias: + +- `side=1`: Long-only. Upper barrier = profit, lower barrier = loss. +- `side=-1`: Short-only. Upper barrier = loss, lower barrier = profit. Labels are flipped. +- `side=0`: Symmetric. Both barriers treated equally. Label is +1 or -1 based on direction. + +### Trailing Stop + +Enable a trailing stop loss that ratchets up as price moves favorably: + +```python +config = LabelingConfig.triple_barrier( + upper_barrier=0.03, + lower_barrier=0.01, + max_holding_period=20, + trailing_stop=True, # Stop loss follows highest price +) +``` + +With `trailing_stop=True`, the lower barrier moves up as the trade moves in favor. This reduces the time spent in losing positions. + +> **Book**: Ch7 `03_label_methods.py` applies triple-barrier labeling to SPY with visualization of barrier touches. + ## ATR-Based Dynamic Barriers -Use volatility-adjusted barriers that adapt to market conditions: +Volatility-adjusted barriers adapt to changing market conditions. The barriers scale with the Average True Range (ATR), so they're wider in volatile markets and tighter in calm markets. ```python -from ml4t.engineer.config import LabelingConfig from ml4t.engineer.labeling import atr_triple_barrier_labels +result = atr_triple_barrier_labels( + data=df, + atr_tp_multiple=2.0, # Take profit at 2x ATR + atr_sl_multiple=1.0, # Stop loss at 1x ATR + atr_period=14, # ATR lookback + max_holding_bars=20, # Can also be "4h" for time-based + side=1, + price_col="close", + timestamp_col="timestamp", + trailing_stop=False, +) + +# Or via LabelingConfig config = LabelingConfig.atr_barrier( + atr_tp_multiple=2.0, + atr_sl_multiple=1.0, atr_period=14, - atr_tp_multiple=2.0, # 2x ATR profit target - atr_sl_multiple=1.0, # 1x ATR stop loss max_holding_period=20, ) +result = atr_triple_barrier_labels(df, config=config) +``` + +### When to Use ATR Barriers + +| Scenario | Recommendation | +|----------|---------------| +| Single asset, stable volatility | Fixed barriers sufficient | +| Multi-asset (different volatility levels) | ATR barriers adapt per asset | +| Regime changes (calm → volatile) | ATR barriers avoid premature stops | +| Futures with varying contract sizes | ATR normalizes across contracts | + +> **Book**: CME Futures case study `02_labels.py` applies ATR barriers on ES, NQ, and CL futures with session-aware horizons. + +## Rolling Percentile Labels + +Adaptive labeling where thresholds are computed from the rolling return distribution. Produces binary long/short signals based on whether forward returns exceed a historical percentile. + +```python +from ml4t.engineer.labeling import rolling_percentile_binary_labels + +result = rolling_percentile_binary_labels( + data=df, + horizon=10, # Forward return horizon (bars or "1h") + percentile=95, # 95th percentile for long signals + direction="long", # "long" or "short" + lookback_window=252 * 24, # ~1 year of hourly bars + price_col="close", + session_col=None, # Session-aware forward returns + min_samples=None, # Minimum samples for percentile + timestamp_col=None, # Required for time-based horizons + tolerance=None, # E.g., "2m" for time-based horizons +) +``` + +### Output Columns + +| Column | Example | Description | +|--------|---------|-------------| +| `forward_return_10` | 0.0123 | Forward return over horizon | +| `threshold_p95_h10` | 0.0089 | Rolling 95th percentile threshold | +| `label_long_p95_h10` | 1 | 1 if return exceeds threshold, 0 otherwise | + +### Multiple Horizons and Percentiles + +Generate labels for multiple combinations simultaneously: + +```python +from ml4t.engineer.labeling import rolling_percentile_multi_labels + +result = rolling_percentile_multi_labels( + data=df, + horizons=[5, 10, 20], + percentiles=[90, 95], + direction="long", + lookback_window=252, +) +# Produces: label_long_p90_h5, label_long_p95_h5, label_long_p90_h10, ... +``` + +> **Book**: Ch7 `03_label_methods.py` compares rolling percentile labels against triple-barrier on SPY. ETFs case study `02_labels.py` uses percentile labels in its production pipeline. + +## Fixed Time Horizon + +The simplest labeling method: compute forward returns over a fixed horizon. Supports both bar-count and time-based horizons. + +```python +from ml4t.engineer.labeling import fixed_time_horizon_labels + +# Bar-based horizon +result = fixed_time_horizon_labels( + data=df, + horizon=10, # 10 bars forward + price_col="close", +) -labels = atr_triple_barrier_labels(df, config=config) +# Time-based horizon +result = fixed_time_horizon_labels( + data=df, + horizon="1h", # 1 hour forward + timestamp_col="timestamp", + price_col="close", +) ``` +Output includes a `forward_return` column. For binary labels, use the `threshold` parameter or `rolling_percentile_binary_labels` for adaptive thresholds. + +## Trend Scanning + +De Prado's trend-scanning method finds the optimal holding period for each observation by fitting linear regressions over a range of horizons and selecting the one with the highest t-statistic. + +```python +from ml4t.engineer.labeling import trend_scanning_labels + +result = trend_scanning_labels( + data=df, + min_horizon=5, + max_horizon=20, + t_value_threshold=2.0, + price_col="close", +) +``` + +Output includes `trend_label` (+1/-1/0), `optimal_horizon`, and `t_statistic`. Observations with |t-statistic| below the threshold receive label 0 (no trend). + +> **Book**: Ch7 `03_label_methods.py` demonstrates trend scanning alongside triple-barrier and percentile methods, showing how the optimal horizon varies with market conditions. + +## Meta-Labeling & Bet Sizing + +Meta-labeling is a two-stage workflow (AFML Chapter 3): + +1. A **primary model** generates directional signals (+1/-1/0) +2. A **meta-model** predicts whether the primary signal will be profitable (1/0) +3. **Bet sizing** converts meta-model probability into position sizes + +### Step 1: Generate Meta-Labels + +```python +from ml4t.engineer.labeling import meta_labels + +meta_result = meta_labels( + data=df, + signal_col="primary_signal", # +1/-1/0 from primary model + return_col="forward_return", # Actual forward returns + threshold=0.0, # Minimum return for "correct" +) +# Adds "meta_label" column: 1 if signal was correct, 0 otherwise +``` + +### Step 2: Train Meta-Model + +Train any classifier on the meta-labels to predict P(primary signal is correct). + +### Step 3: Compute Bet Sizes + +```python +from ml4t.engineer.labeling import compute_bet_size, apply_meta_model + +# Low-level: get bet size expression +bet_expr = compute_bet_size( + probability="meta_probability", # Column name or pl.Expr + method="sigmoid", # "linear" | "sigmoid" | "discrete" + scale=5.0, # Sigmoid steepness + threshold=0.5, # Minimum probability to bet +) + +# High-level: apply meta-model to size positions +result = apply_meta_model( + data=df, + primary_signal_col="signal", + meta_probability_col="meta_prob", + bet_size_method="sigmoid", + scale=5.0, + threshold=0.5, + output_col="sized_signal", # signal * bet_size +) +``` + +### Bet Sizing Methods + +| Method | Formula | When to Use | +|--------|---------|-------------| +| `"linear"` | `max(0, p - threshold) / (1 - threshold)` | Simple, interpretable | +| `"sigmoid"` | `2 / (1 + exp(-scale * (p - 0.5))) - 1` | Smooth, differentiable | +| `"discrete"` | `1 if p >= threshold else 0` | Binary position sizing | + +> **Book**: Ch7 `03_label_methods.py` implements the complete meta-labeling workflow: primary model signals → meta-labels → bet sizing on SPY. + +## Calendar-Aware Labels + +For futures and other instruments with defined trading sessions, calendar-aware labeling respects session boundaries. A label that would span an overnight gap is handled correctly. + +```python +from ml4t.engineer.labeling import calendar_aware_labels + +result = calendar_aware_labels( + data=df, + config=config, # LabelingConfig + calendar="CME_Equity", # "NYSE", "CME_Equity", etc. + price_col="close", + timestamp_col="timestamp", +) +``` + +The calendar prevents forward returns from crossing session breaks (e.g., CME overnight gap from 4:00 PM to 5:00 PM CT). + +## Sample Weighting + +AFML Chapter 4 addresses the problem of overlapping labels creating correlated samples. The library provides the full toolkit: + +### Label Uniqueness + +```python +from ml4t.engineer.labeling import ( + build_concurrency, + calculate_label_uniqueness, + calculate_sample_weights, +) + +# Count how many labels overlap each bar +concurrency = build_concurrency( + event_indices=starts, # Label start indices + label_indices=ends, # Label end indices + n_bars=len(df), +) + +# Average uniqueness per label (range [0, 1]) +uniqueness = calculate_label_uniqueness( + event_indices=starts, + label_indices=ends, + n_bars=len(df), +) + +# Combine uniqueness with return magnitude +weights = calculate_sample_weights( + uniqueness=uniqueness, + returns=returns_array, + weight_scheme="returns_uniqueness", + # Options: "returns_uniqueness", "uniqueness_only", "returns_only", "equal" +) +``` + +### Sequential Bootstrap + +The sequential bootstrap (AFML Chapter 4) draws samples while accounting for label overlap, producing a more independent training set: + +```python +from ml4t.engineer.labeling import sequential_bootstrap + +selected = sequential_bootstrap( + starts=start_indices, + ends=end_indices, + n_bars=len(df), + n_draws=1000, # Number of samples to draw + with_replacement=True, + random_state=42, +) +# selected: array of indices for training +``` + +### Integrated Computation + +Triple-barrier labels can compute uniqueness and weights in a single call: + +```python +result = triple_barrier_labels( + df, + config=config, + calculate_uniqueness=True, + uniqueness_weight_scheme="returns_uniqueness", +) +# Result includes label_uniqueness and sample_weight columns +``` + +### Label Statistics + +Quick summary of label balance: + +```python +from ml4t.engineer.labeling import compute_label_statistics + +stats = compute_label_statistics(df, label_col="label") +# Returns: {"n_samples", "n_positive", "n_negative", "n_neutral", +# "positive_ratio", "negative_ratio", "neutral_ratio"} +``` + +> **Book**: Ch7 `03_label_methods.py` demonstrates sequential bootstrap applied to triple-barrier labels, showing how it reduces effective sample size while improving independence. + +## Time-Based Durations + +All labeling functions that accept `max_holding_period` or `horizon` support duration strings in addition to bar counts: + +```python +# Bar-based (integer) +config = LabelingConfig.triple_barrier(max_holding_period=20) + +# Time-based (duration string) +config = LabelingConfig.triple_barrier(max_holding_period="4h") +config = LabelingConfig.triple_barrier(max_holding_period="1d") +config = LabelingConfig.triple_barrier(max_holding_period="30m") +``` + +### Supported Duration Formats + +| Format | Example | Meaning | +|--------|---------|---------| +| Minutes | `"30m"` | 30 minutes | +| Hours | `"4h"` | 4 hours | +| Days | `"1d"` | 1 day | +| Combined | `"1h30m"` | 1 hour 30 minutes | + +### Utility Functions + +```python +from ml4t.engineer.labeling.utils import ( + is_duration_string, # Check: is_duration_string("4h") → True + parse_duration, # Parse: parse_duration("1h30m") → timedelta(hours=1, minutes=30) + time_horizon_to_bars, # Convert to per-event bar counts using timestamps + get_future_price_at_time, # Price at exact time offset +) +``` + +Time-based horizons require a `timestamp_col` in the input DataFrame. + ## Performance -- **Speed**: 50,000 labels/second -- **Memory**: Efficient vectorized implementation -- **Accuracy**: Exact match with López de Prado's reference +- **Speed**: ~50,000 labels/second (Numba-accelerated) +- **Memory**: Efficient vectorized implementation via Polars +- **Accuracy**: Exact match with AFML reference (validated at 1e-10 tolerance against mlfinpy) ## Best Practices -1. **Avoid overlapping labels**: Use `min_return` to filter small moves -2. **Handle class imbalance**: Triple-barrier often creates imbalanced labels -3. **Account for transaction costs**: Barriers should exceed expected costs +1. **Match barriers to transaction costs**: Barriers should exceed expected round-trip costs. A 0.1% barrier with 0.05% commission leaves little net profit. + +2. **Handle class imbalance**: Triple-barrier often creates imbalanced labels (many vertical barrier hits). Check with `compute_label_statistics()` and consider adjusting barrier levels or using sample weights. + +3. **Prevent leakage with sample weighting**: Overlapping labels create correlated training samples. Use `calculate_uniqueness=True` or `sequential_bootstrap()` to address this. + +4. **Use ATR barriers for multi-asset**: Fixed barriers work for single-asset studies but fail across assets with different volatility levels. + +5. **Time-based horizons for irregular data**: If your bars are not equally spaced (e.g., volume bars), use duration strings (`"4h"`) instead of bar counts to ensure consistent holding periods. + +## References + +- Lopez de Prado, M. (2018). *Advances in Financial Machine Learning*. Wiley. Chapters 3-4. +- Lopez de Prado, M. (2020). *Machine Learning for Asset Managers*. Cambridge. diff --git a/docs/user-guide/ml-readiness.md b/docs/user-guide/ml-readiness.md index 907b6ad..2a4a11b 100644 --- a/docs/user-guide/ml-readiness.md +++ b/docs/user-guide/ml-readiness.md @@ -2,6 +2,8 @@ This guide explains the `normalized` field in feature metadata and how to prepare features for machine learning models. +> **Book**: *ML for Trading, 3rd ed.* — Ch8 `01_price_volume_features.py` compares normalized vs non-normalized features on real ETF data, including preprocessing strategies for each type. + ## Overview Features in ml4t-engineer have a `normalized` metadata field that indicates whether a feature produces bounded outputs suitable for direct ML consumption: diff --git a/docs/user-guide/preprocessing.md b/docs/user-guide/preprocessing.md new file mode 100644 index 0000000..3e59efa --- /dev/null +++ b/docs/user-guide/preprocessing.md @@ -0,0 +1,171 @@ +# Preprocessing + +ML4T Engineer provides sklearn-compatible scalers built on Polars for leakage-safe feature preprocessing. + +## Scalers + +All scalers follow the sklearn pattern: `fit()` on training data, `transform()` on any data. This prevents information leakage from test data into training. + +### StandardScaler + +Z-score normalization: output has mean=0, std=1. + +```python +from ml4t.engineer.preprocessing import StandardScaler + +scaler = StandardScaler( + columns=None, # None = all numeric columns + with_mean=True, # Center to zero mean + with_std=True, # Scale to unit variance + ddof=1, # Delta degrees of freedom +) + +# Fit on training data +train_scaled = scaler.fit_transform(train_df) + +# Transform test data using training statistics +test_scaled = scaler.transform(test_df) +``` + +Best for: Approximately Gaussian features. Default choice for most ML models. + +### MinMaxScaler + +Scale features to a bounded range (default [0, 1]). + +```python +from ml4t.engineer.preprocessing import MinMaxScaler + +scaler = MinMaxScaler( + columns=None, + feature_range=(0.0, 1.0), # Target range +) + +train_scaled = scaler.fit_transform(train_df) +test_scaled = scaler.transform(test_df) +``` + +Best for: Neural networks expecting bounded input, or when preserving zero values matters. + +### RobustScaler + +IQR-based scaling that's resistant to outliers. + +```python +from ml4t.engineer.preprocessing import RobustScaler + +scaler = RobustScaler( + columns=None, + with_centering=True, # Subtract median + with_scaling=True, # Scale by IQR + quantile_range=(25.0, 75.0), # IQR range +) + +train_scaled = scaler.fit_transform(train_df) +test_scaled = scaler.transform(test_df) +``` + +Best for: Financial data with fat tails, outliers, or extreme values. + +### When to Use Each Scaler + +| Scaler | Use When | Sensitive To | +|--------|----------|-------------| +| `StandardScaler` | Data is approximately Gaussian | Outliers | +| `MinMaxScaler` | Need bounded 0-1 range | Outliers | +| `RobustScaler` | Data has outliers or fat tails | Nothing (robust) | + +For financial data, `RobustScaler` is generally the safest default due to fat-tailed return distributions. + +## Leakage Prevention + +The critical rule: **fit on training data only, transform everything**. + +```python +# CORRECT: fit on train, transform both +scaler = StandardScaler() +X_train = scaler.fit_transform(train_df) +X_test = scaler.transform(test_df) # Uses train statistics + +# WRONG: fitting on all data leaks test information +scaler = StandardScaler() +X_all = scaler.fit_transform(all_data) # Leaks test statistics! +``` + +### Scaler State + +After fitting, inspect the learned statistics: + +```python +scaler.is_fitted # True after fit/fit_transform +scaler.fitted_columns # ["rsi", "macd", "atr", ...] +scaler.statistics # {"rsi": {"mean": 52.3, "std": 15.1}, ...} +``` + +### Serialization + +Save and reload fitted scalers: + +```python +# Save +state = scaler.to_dict() + +# Reload +scaler = StandardScaler.from_dict(state) +``` + +### Cloning + +Create an unfitted copy with the same parameters: + +```python +new_scaler = scaler.clone() # Same params, unfitted +``` + +## PreprocessingPipeline + +For multi-step preprocessing, chain transforms: + +```python +from ml4t.engineer.preprocessing import PreprocessingPipeline + +pipeline = PreprocessingPipeline.from_recommendations({ + "rsi_14": {"transform": "standardize", "confidence": 0.9}, + "volume": {"transform": "log", "confidence": 0.8}, + "returns": {"transform": "winsorize", "confidence": 0.85}, +}) + +train_transformed = pipeline.fit_transform(train_df) +test_transformed = pipeline.transform(test_df) +``` + +### Available Transform Types + +| Transform | Description | +|-----------|-------------| +| `NONE` | No transformation | +| `LOG` | Log transform (for skewed data) | +| `SQRT` | Square root transform | +| `STANDARDIZE` | Z-score normalization | +| `NORMALIZE` | Min-max scaling | +| `WINSORIZE` | Clip extreme values | +| `DIFF` | First difference | + +## Integration with MLDatasetBuilder + +The preprocessing module integrates with `MLDatasetBuilder` for a leakage-safe end-to-end workflow. See the [Dataset Builder guide](dataset-builder.md) for details. + +```python +from ml4t.engineer import create_dataset_builder + +builder = create_dataset_builder( + features=features_df, + labels=labels_series, + scaler="robust", # "standard", "minmax", "robust", or None +) + +# Scaling happens automatically during train/test split +X_train, X_test, y_train, y_test = builder.train_test_split(train_size=0.8) +``` + +> **Book**: *ML for Trading, 3rd ed.* — Ch7 `02_preprocessing_pipeline.py` demonstrates split-aware preprocessing with StandardScaler in a feature preparation pipeline. See also the [ML-Readiness guide](ml-readiness.md) for which features need preprocessing vs. which are ML-ready out of the box. diff --git a/pyproject.toml b/pyproject.toml index db22366..e16f00e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -135,7 +135,6 @@ dev = [ "pytest-benchmark>=4.0.0", "hypothesis>=6.80.0", "ruff>=0.1.0", - "mypy>=1.5.0", "ipython>=8.14.0", "ipdb>=0.13.0", "pre-commit>=3.3.0", @@ -174,7 +173,6 @@ all = [ "pytest-benchmark>=4.0.0", "hypothesis>=6.80.0", "ruff>=0.1.0", - "mypy>=1.5.0", "ipython>=8.14.0", "ipdb>=0.13.0", "pre-commit>=3.3.0", @@ -205,6 +203,7 @@ dev = [ "ty", "pre-commit>=3.3.0", "twine>=6.0.0", + "pandas-market-calendars>=4.0.0", ] [tool.pytest.ini_options] @@ -215,6 +214,7 @@ python_functions = ["test_*"] addopts = [ "-ra", "--strict-markers", + "-m", "not perf", "--capture=no", # Disable capture to avoid crash with numba/numpy on Python 3.13 "--cov=ml4t.engineer", "--cov-report=term-missing", @@ -224,7 +224,7 @@ markers = [ "slow: marks tests as slow", "integration: marks tests as integration tests", "benchmark: marks benchmark tests", - "performance: marks performance tests", + "perf: marks performance comparison tests (excluded from default runs, use pytest -m perf)", "validation: marks tests that validate against TA-Lib", "property: marks property-based tests using hypothesis", ] @@ -257,22 +257,6 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "tests/*" = ["E402", "ARG002", "B017", "F841"] -[tool.mypy] -python_version = "3.11" -strict = true -warn_return_any = true -warn_unused_ignores = true -disallow_untyped_defs = true -disallow_any_unimported = false -no_implicit_optional = true -check_untyped_defs = true -show_error_codes = true -warn_redundant_casts = true -ignore_missing_imports = true -namespace_packages = true -explicit_package_bases = true -mypy_path = "src" - [tool.ty.environment] python-version = "3.11" root = ["src"] diff --git a/src/ml4t/engineer/__init__.py b/src/ml4t/engineer/__init__.py index ee543ca..2839815 100644 --- a/src/ml4t/engineer/__init__.py +++ b/src/ml4t/engineer/__init__.py @@ -1,8 +1,8 @@ """ml4t-engineer - A Financial Machine Learning Feature Engineering Library. ml4t-engineer is a comprehensive FML stack designed for correctness, reproducibility, -and performance. It provides tools for feature engineering, labeling, and validation -of financial machine learning models. +and performance. It provides tools for feature engineering, labeling, and preprocessing +for financial machine learning models. Agent Navigation: This package includes AGENT.md files for AI agent navigation. @@ -20,12 +20,9 @@ discovery, features, labeling, - pipeline, preprocessing, relationships, store, - validation, - visualization, ) from .api import compute_features from .dataset import ( @@ -129,10 +126,7 @@ def get_agent_docs() -> dict[str, _Path]: "discovery", "features", "labeling", - "pipeline", "preprocessing", "relationships", "store", - "validation", - "visualization", ] diff --git a/src/ml4t/engineer/bars/imbalance.py b/src/ml4t/engineer/bars/imbalance.py index 76f293a..ed31d32 100644 --- a/src/ml4t/engineer/bars/imbalance.py +++ b/src/ml4t/engineer/bars/imbalance.py @@ -19,8 +19,6 @@ E[v] = unconditional mean volume per tick """ -import warnings - import numpy as np import numpy.typing as npt import polars as pl @@ -544,8 +542,6 @@ class ImbalanceBarSampler(BarSampler): ---------- expected_ticks_per_bar : int Expected number of ticks per bar (used to initialize E[T]) - initial_expectation : float, optional - DEPRECATED. Use expected_ticks_per_bar instead. alpha : float, default 0.1 EWMA decay factor for updating expectations initial_p_buy : float, default 0.5 @@ -570,26 +566,10 @@ class ImbalanceBarSampler(BarSampler): def __init__( self, expected_ticks_per_bar: int, - initial_expectation: float | None = None, alpha: float = 0.1, initial_p_buy: float = 0.5, min_bars_warmup: int = 10, ): - """Initialize imbalance bar sampler. - - Parameters - ---------- - expected_ticks_per_bar : int - Expected number of ticks per bar - initial_expectation : float, optional - DEPRECATED. Use expected_ticks_per_bar instead. - alpha : float, default 0.1 - EWMA decay factor - initial_p_buy : float, default 0.5 - Initial buy probability P[b=1] - min_bars_warmup : int, default 10 - Number of bars before starting EWMA updates - """ if expected_ticks_per_bar <= 0: raise ValueError("expected_ticks_per_bar must be positive") @@ -602,16 +582,7 @@ def __init__( if min_bars_warmup < 0: raise ValueError("min_bars_warmup must be non-negative") - if initial_expectation is not None: - warnings.warn( - "initial_expectation is deprecated and ignored. " - "The AFML threshold E[T] × |2v⁺ - E[v]| is computed dynamically.", - DeprecationWarning, - stacklevel=2, - ) - self.expected_ticks_per_bar = expected_ticks_per_bar - self.initial_expectation = initial_expectation # Keep for backward compat self.alpha = alpha self.initial_p_buy = initial_p_buy self.min_bars_warmup = min_bars_warmup diff --git a/src/ml4t/engineer/bars/run.py b/src/ml4t/engineer/bars/run.py index f08fe7f..5c98912 100644 --- a/src/ml4t/engineer/bars/run.py +++ b/src/ml4t/engineer/bars/run.py @@ -26,8 +26,6 @@ Based on Advances in Financial Machine Learning by Marcos López de Prado. """ -import warnings - import numpy as np import numpy.typing as npt import polars as pl @@ -179,8 +177,6 @@ class TickRunBarSampler(BarSampler): ---------- expected_ticks_per_bar : int Expected number of ticks per bar (used to initialize E[T]) - initial_run_expectation : int, optional - DEPRECATED. Use expected_ticks_per_bar instead. alpha : float, default 0.1 EWMA decay factor for updating expectations initial_p_buy : float, default 0.5 @@ -202,7 +198,6 @@ class TickRunBarSampler(BarSampler): def __init__( self, expected_ticks_per_bar: int, - initial_run_expectation: int | None = None, alpha: float = 0.1, initial_p_buy: float = 0.5, min_bars_warmup: int = 10, @@ -216,16 +211,7 @@ def __init__( if min_bars_warmup < 0: raise ValueError("min_bars_warmup must be non-negative") - if initial_run_expectation is not None: - warnings.warn( - "initial_run_expectation is deprecated and ignored. " - "The AFML threshold E[T] × max{P[b=1], 1-P[b=1]} is computed dynamically.", - DeprecationWarning, - stacklevel=2, - ) - self.expected_ticks_per_bar = expected_ticks_per_bar - self.initial_run_expectation = initial_run_expectation # Keep for compat self.alpha = alpha self.initial_p_buy = initial_p_buy self.min_bars_warmup = min_bars_warmup @@ -395,8 +381,6 @@ class VolumeRunBarSampler(BarSampler): ---------- expected_ticks_per_bar : int Expected number of ticks per bar - initial_run_expectation : float, optional - DEPRECATED. Threshold is computed dynamically. alpha : float, default 0.1 EWMA decay factor initial_p_buy : float, default 0.5 @@ -413,7 +397,6 @@ class VolumeRunBarSampler(BarSampler): def __init__( self, expected_ticks_per_bar: int, - initial_run_expectation: float | None = None, alpha: float = 0.1, initial_p_buy: float = 0.5, min_bars_warmup: int = 10, @@ -427,16 +410,7 @@ def __init__( if min_bars_warmup < 0: raise ValueError("min_bars_warmup must be non-negative") - if initial_run_expectation is not None: - warnings.warn( - "initial_run_expectation is deprecated and ignored. " - "The AFML threshold is computed dynamically.", - DeprecationWarning, - stacklevel=2, - ) - self.expected_ticks_per_bar = expected_ticks_per_bar - self.initial_run_expectation = initial_run_expectation self.alpha = alpha self.initial_p_buy = initial_p_buy self.min_bars_warmup = min_bars_warmup @@ -592,8 +566,6 @@ class DollarRunBarSampler(BarSampler): ---------- expected_ticks_per_bar : int Expected number of ticks per bar - initial_run_expectation : float, optional - DEPRECATED. Threshold is computed dynamically. alpha : float, default 0.1 EWMA decay factor initial_p_buy : float, default 0.5 @@ -610,7 +582,6 @@ class DollarRunBarSampler(BarSampler): def __init__( self, expected_ticks_per_bar: int, - initial_run_expectation: float | None = None, alpha: float = 0.1, initial_p_buy: float = 0.5, min_bars_warmup: int = 10, @@ -624,16 +595,7 @@ def __init__( if min_bars_warmup < 0: raise ValueError("min_bars_warmup must be non-negative") - if initial_run_expectation is not None: - warnings.warn( - "initial_run_expectation is deprecated and ignored. " - "The AFML threshold is computed dynamically.", - DeprecationWarning, - stacklevel=2, - ) - self.expected_ticks_per_bar = expected_ticks_per_bar - self.initial_run_expectation = initial_run_expectation self.alpha = alpha self.initial_p_buy = initial_p_buy self.min_bars_warmup = min_bars_warmup diff --git a/src/ml4t/engineer/bars/vectorized.py b/src/ml4t/engineer/bars/vectorized.py index 4abd405..f5a167b 100644 --- a/src/ml4t/engineer/bars/vectorized.py +++ b/src/ml4t/engineer/bars/vectorized.py @@ -385,8 +385,6 @@ class ImbalanceBarSamplerVectorized(BarSampler): ---------- expected_ticks_per_bar : int Expected number of ticks per bar (initializes E[T]) - initial_expectation : float, optional - DEPRECATED. The AFML threshold is computed dynamically. alpha : float, default 0.1 EWMA decay factor for updating expectations initial_p_buy : float, default 0.5 @@ -398,7 +396,6 @@ class ImbalanceBarSamplerVectorized(BarSampler): def __init__( self, expected_ticks_per_bar: int, - initial_expectation: float | None = None, alpha: float = 0.1, initial_p_buy: float = 0.5, min_bars_warmup: int = 10, @@ -413,7 +410,6 @@ def __init__( raise ValueError("min_bars_warmup must be non-negative") self.expected_ticks_per_bar = expected_ticks_per_bar - self.initial_expectation = initial_expectation # Deprecated, kept for backward compat self.alpha = alpha self.initial_p_buy = initial_p_buy self.min_bars_warmup = min_bars_warmup diff --git a/src/ml4t/engineer/config/__init__.py b/src/ml4t/engineer/config/__init__.py index c214683..d9a7e4e 100644 --- a/src/ml4t/engineer/config/__init__.py +++ b/src/ml4t/engineer/config/__init__.py @@ -4,39 +4,12 @@ - **Labeling**: Triple barrier, ATR barrier, fixed horizon, trend scanning - **Preprocessing**: Standard, MinMax, Robust scalers with create_scaler() -- **Feature Diagnostics**: Stationarity, ACF, volatility, distribution -- **Cross-Feature Analysis**: Correlation, PCA, clustering, redundancy -- **Feature-Outcome Analysis**: IC, classification, thresholds, ML diagnostics +- **Data Contract**: Schema validation for input data +- **Experiment**: Experiment configuration and serialization -D06 Pattern Support: - This module supports the D06 configuration pattern with single-level nesting: - - Primary configs use `*Config` naming (e.g., `EngineerConfig`) - - Nested settings use `*Settings` naming (e.g., `StationaritySettings`) - - Both patterns are fully supported via aliases - -Examples: - Quick start with defaults: - - >>> from ml4t.engineer.config import EngineerConfig - >>> config = EngineerConfig() - - Use presets: - - >>> config = EngineerConfig.for_quick_analysis() - >>> config = EngineerConfig.for_research() - >>> config = EngineerConfig.for_production() - - Custom configuration: - - >>> config = EngineerConfig( - ... module_a=ModuleAConfig( - ... stationarity=StationaritySettings(significance_level=0.01) - ... ) - ... ) - - Load from YAML: - - >>> config = EngineerConfig.from_yaml("config.yaml") +Note: + Feature evaluation configs (StationarityConfig, ACFConfig, etc.) have moved + to ``ml4t-diagnostic``. Install with: ``pip install ml4t-diagnostic`` """ from ml4t.engineer.config.base import ( @@ -50,65 +23,14 @@ load_experiment_config, save_experiment_config, ) -from ml4t.engineer.config.feature_config import ( - ACFConfig, - BinaryClassificationConfig, - ClusteringConfig, - CorrelationConfig, - DistributionConfig, - FeatureEvaluatorConfig, - ICConfig, - MLDiagnosticsConfig, - ModuleAConfig, - ModuleBConfig, - ModuleCConfig, - PCAConfig, - RedundancyConfig, - StationarityConfig, - ThresholdAnalysisConfig, - VolatilityConfig, -) from ml4t.engineer.config.labeling import LabelingConfig from ml4t.engineer.config.preprocessing_config import PreprocessingConfig -# ============================================================================= -# D06 Pattern Aliases - Settings Classes -# ============================================================================= -# These aliases provide compatibility with the diagnostic library's D06 pattern -# where nested configuration classes use *Settings naming. - -StationaritySettings = StationarityConfig -ACFSettings = ACFConfig -VolatilitySettings = VolatilityConfig -DistributionSettings = DistributionConfig -CorrelationSettings = CorrelationConfig -PCASettings = PCAConfig -ClusteringSettings = ClusteringConfig -RedundancySettings = RedundancyConfig -ICSettings = ICConfig -BinaryClassificationSettings = BinaryClassificationConfig -ThresholdAnalysisSettings = ThresholdAnalysisConfig -MLDiagnosticsSettings = MLDiagnosticsConfig - -# ============================================================================= -# D06 Pattern Aliases - Top-Level Configs -# ============================================================================= - -# Primary alias - EngineerConfig is the D06-style name -EngineerConfig = FeatureEvaluatorConfig - -# DiagnosticConfig alias for symmetry with ml4t.diagnostic -DiagnosticConfig = FeatureEvaluatorConfig - -# RuntimeConfig is the D06-style name for computational settings -RuntimeConfig = ComputationalConfig - __all__ = [ # Base configs "BaseConfig", "StatisticalTestConfig", - "RuntimeConfig", - "ComputationalConfig", # Backward compatibility + "ComputationalConfig", # Labeling and preprocessing configs "LabelingConfig", "DataContractConfig", @@ -117,51 +39,4 @@ "ExperimentConfig", "load_experiment_config", "save_experiment_config", - # Primary config (D06 pattern) - "EngineerConfig", - "DiagnosticConfig", # Alias for symmetry - # Feature evaluation (original names) - "FeatureEvaluatorConfig", - "ModuleAConfig", - "ModuleBConfig", - "ModuleCConfig", - # Settings classes (D06 pattern - *Settings naming) - "StationaritySettings", - "ACFSettings", - "VolatilitySettings", - "DistributionSettings", - "CorrelationSettings", - "PCASettings", - "ClusteringSettings", - "RedundancySettings", - "ICSettings", - "BinaryClassificationSettings", - "ThresholdAnalysisSettings", - "MLDiagnosticsSettings", - # Original config names (backward compatibility) - "StationarityConfig", - "ACFConfig", - "VolatilityConfig", - "DistributionConfig", - "CorrelationConfig", - "PCAConfig", - "ClusteringConfig", - "RedundancyConfig", - "ICConfig", - "BinaryClassificationConfig", - "ThresholdAnalysisConfig", - "MLDiagnosticsConfig", ] - -_REMOVED_EXPORTS = { - "BarrierLabelingConfig": ( - "ml4t.engineer.config.BarrierLabelingConfig has been removed. " - "Use ml4t.engineer.config.LabelingConfig instead." - ) -} - - -def __getattr__(name: str) -> object: - if name in _REMOVED_EXPORTS: - raise ImportError(_REMOVED_EXPORTS[name]) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ml4t/engineer/config/feature_config.py b/src/ml4t/engineer/config/feature_config.py deleted file mode 100644 index b4d310e..0000000 --- a/src/ml4t/engineer/config/feature_config.py +++ /dev/null @@ -1,946 +0,0 @@ -# ruff: noqa: UP006, UP045 -"""Feature evaluation configuration (Modules A, B, C). - -Exports: - Module A (Feature Diagnostics): - StationarityConfig - ADF, KPSS, PP test settings - ACFConfig - Autocorrelation analysis settings - VolatilityConfig - Volatility clustering detection - DistributionConfig - Distribution analysis settings - ModuleAConfig - Combined Module A configuration - - Module B (Cross-Feature Analysis): - CorrelationConfig - Correlation matrix settings - PCAConfig - Principal component analysis - ClusteringConfig - Feature clustering - RedundancyConfig - Redundancy detection - ModuleBConfig - Combined Module B configuration - - Module C (Feature-Outcome): - ICConfig - Information coefficient analysis - BinaryClassificationConfig - Classification metrics - ThresholdAnalysisConfig - Threshold optimization - MLDiagnosticsConfig - SHAP and importance settings - ModuleCConfig - Combined Module C configuration - - Main: - FeatureEvaluatorConfig - Master configuration for all modules - -This module defines configuration for: -- **Module A**: Feature diagnostics (stationarity, ACF, volatility, distribution) -- **Module B**: Cross-feature analysis (correlation, PCA, clustering, redundancy) -- **Module C**: Feature-outcome relationships (IC, classification, thresholds, ML diagnostics) -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Literal - -from pydantic import Field, field_validator, model_validator - -from ml4t.engineer.config.base import BaseConfig, StatisticalTestConfig -from ml4t.engineer.config.validation import ( - ClusteringMethod, - CorrelationMethod, - DistanceMetric, - DriftDetectionMethod, - LinkageMethod, - NonNegativeInt, - NormalityTest, - OutlierMethod, - PositiveFloat, - PositiveInt, - Probability, - RegressionType, - ThresholdOptimizationTarget, - VolatilityClusterMethod, - validate_min_max_range, -) - -# ============================================================================= -# Module A: Feature Diagnostics -# ============================================================================= - - -class StationarityConfig(StatisticalTestConfig): - """Configuration for stationarity testing. - - Tests whether time series are stationary (constant mean/variance over time). - Non-stationary series should typically be differenced or detrended before use - in forecasting models. - - Attributes: - enabled: Run stationarity tests - adf_enabled: Run Augmented Dickey-Fuller test - kpss_enabled: Run KPSS test - pp_enabled: Run Phillips-Perron test - adf_regression: Regression type for ADF ("c", "ct", "ctt", "n") - kpss_regression: Regression type for KPSS ("c", "ct") - pp_regression: Regression type for PP ("c", "ct", "ctt", "n") - max_lag: Maximum lag for tests ("auto" or positive int) - significance_level: Significance level for hypothesis tests - - Examples: - >>> # Default: ADF + KPSS at 5% significance - >>> config = StationarityConfig() - - >>> # Custom: Only ADF at 1% significance - >>> config = StationarityConfig( - ... adf_enabled=True, - ... kpss_enabled=False, - ... pp_enabled=False, - ... significance_level=0.01 - ... ) - - References: - - Dickey, D.A. and Fuller, W.A. (1979). "Distribution of the estimators for - autoregressive time series with a unit root." JASA. - - Kwiatkowski et al. (1992). "Testing the null hypothesis of stationarity - against the alternative of a unit root." Journal of Econometrics. - """ - - adf_enabled: bool = Field(True, description="Run Augmented Dickey-Fuller test") - kpss_enabled: bool = Field(True, description="Run KPSS test") - pp_enabled: bool = Field(False, description="Run Phillips-Perron test (similar to ADF)") - adf_regression: RegressionType = Field( - RegressionType.CONSTANT, description="ADF regression type: c, ct, ctt, or n" - ) - kpss_regression: Literal["c", "ct"] = Field("c", description="KPSS regression type: c or ct") - pp_regression: RegressionType = Field( - RegressionType.CONSTANT, description="PP regression type: c, ct, ctt, or n" - ) - max_lag: Literal["auto"] | PositiveInt = Field( - "auto", description="Maximum lag for tests (auto or positive int)" - ) - - @model_validator(mode="after") - def check_at_least_one_test(self) -> StationarityConfig: - """Ensure at least one test is enabled.""" - if not (self.adf_enabled or self.kpss_enabled or self.pp_enabled): - raise ValueError("At least one stationarity test must be enabled") - return self - - -class ACFConfig(BaseConfig): - """Configuration for autocorrelation function (ACF) and partial ACF (PACF) analysis. - - Analyzes temporal dependencies in time series to detect: - - Serial correlation (autocorrelation) - - Lag structure for AR/MA models - - Periodicity and cycles - - Attributes: - enabled: Run ACF/PACF analysis - n_lags: Number of lags to compute (auto or positive int) - alpha: Significance level for confidence bands - compute_pacf: Also compute partial autocorrelation - pacf_method: Method for PACF ("yw", "ols", "mle") - use_fft: Use FFT for ACF computation (faster for long series) - - Examples: - >>> # Default: 40 lags with 95% confidence - >>> config = ACFConfig() - - >>> # Custom: 100 lags with 99% confidence - >>> config = ACFConfig(n_lags=100, alpha=0.01) - """ - - enabled: bool = Field(True, description="Run ACF/PACF analysis") - n_lags: Literal["auto"] | PositiveInt = Field( - 40, description="Number of lags (auto = min(10*log10(n), n//2))" - ) - alpha: Probability = Field(0.05, description="Significance level for confidence bands") - compute_pacf: bool = Field(True, description="Also compute partial autocorrelation") - pacf_method: Literal["yw", "ols", "mle"] = Field( - "yw", description="PACF method: yw (Yule-Walker), ols, or mle" - ) - use_fft: bool = Field(True, description="Use FFT for ACF computation (faster)") - - -class VolatilityConfig(BaseConfig): - """Configuration for volatility analysis. - - Analyzes volatility patterns to detect: - - Volatility clustering (GARCH effects) - - Heteroscedasticity - - Regime changes - - Attributes: - enabled: Run volatility analysis - window_sizes: Rolling window sizes for volatility estimation - detect_clustering: Test for volatility clustering (GARCH effects) - cluster_method: Method for cluster detection ("ljung_box", "engle_arch") - significance_level: Significance level for clustering tests - compute_rolling_vol: Compute rolling volatility estimates - - Examples: - >>> # Default: 21-day rolling with cluster detection - >>> config = VolatilityConfig() - - >>> # Custom: Multiple windows without clustering - >>> config = VolatilityConfig( - ... window_sizes=[10, 21, 63], - ... detect_clustering=False - ... ) - """ - - enabled: bool = Field(True, description="Run volatility analysis") - window_sizes: list[PositiveInt] = Field( - default_factory=lambda: [21], description="Rolling window sizes for volatility" - ) - detect_clustering: bool = Field( - True, description="Test for volatility clustering (GARCH effects)" - ) - cluster_method: VolatilityClusterMethod = Field( - VolatilityClusterMethod.LJUNG_BOX, - description="Clustering detection method: ljung_box or engle_arch", - ) - significance_level: Probability = Field( - 0.05, description="Significance level for clustering tests" - ) - compute_rolling_vol: bool = Field(True, description="Compute rolling volatility estimates") - - @field_validator("window_sizes") - @classmethod - def check_window_sizes(cls, v: list[int]) -> list[int]: - """Ensure window sizes are positive and reasonable.""" - if not v: - raise ValueError("Must specify at least one window size") - if any(w < 2 for w in v): - raise ValueError("Window sizes must be >= 2") - return sorted(v) # Sort for consistent ordering - - -class DistributionConfig(BaseConfig): - """Configuration for distribution analysis. - - Analyzes distributional properties: - - Normality (critical for many statistical tests) - - Moments (mean, std, skew, kurtosis) - - Outliers - - Attributes: - enabled: Run distribution analysis - test_normality: Test for normality - normality_tests: Which normality tests to run - compute_moments: Compute higher moments (skew, kurtosis) - detect_outliers: Detect outliers - outlier_method: Outlier detection method - outlier_threshold: Z-score threshold for outlier detection - - Examples: - >>> # Default: Normality + moments - >>> config = DistributionConfig() - - >>> # Custom: Full analysis with outlier detection - >>> config = DistributionConfig( - ... normality_tests=[NormalityTest.SHAPIRO, NormalityTest.JARQUE_BERA], - ... detect_outliers=True, - ... outlier_method=OutlierMethod.ISOLATION_FOREST - ... ) - """ - - enabled: bool = Field(True, description="Run distribution analysis") - test_normality: bool = Field(True, description="Test for normality") - normality_tests: list[NormalityTest] = Field( - default_factory=lambda: [NormalityTest.JARQUE_BERA], - description="Normality tests to run", - ) - compute_moments: bool = Field(True, description="Compute higher moments (skew, kurtosis)") - detect_outliers: bool = Field(False, description="Detect outliers (can be expensive)") - outlier_method: OutlierMethod = Field( - OutlierMethod.ZSCORE, description="Outlier detection method" - ) - outlier_threshold: PositiveFloat = Field( - 3.0, description="Z-score threshold for outlier detection" - ) - - -class ModuleAConfig(BaseConfig): - """Configuration for Module A: Feature Diagnostics. - - Analyzes individual feature properties: - - Stationarity (unit roots, trend) - - Temporal structure (autocorrelation) - - Volatility (clustering, heteroscedasticity) - - Distribution (normality, outliers) - - Examples: - >>> # Default: All diagnostics enabled - >>> config = ModuleAConfig() - - >>> # Custom: Only stationarity and ACF - >>> config = ModuleAConfig( - ... stationarity=StationarityConfig(), - ... acf=ACFConfig(), - ... volatility=VolatilityConfig(enabled=False), - ... distribution=DistributionConfig(enabled=False) - ... ) - """ - - stationarity: StationarityConfig = Field( - default_factory=StationarityConfig, description="Stationarity testing configuration" - ) - acf: ACFConfig = Field(default_factory=ACFConfig, description="ACF/PACF configuration") - volatility: VolatilityConfig = Field( - default_factory=VolatilityConfig, description="Volatility analysis configuration" - ) - distribution: DistributionConfig = Field( - default_factory=DistributionConfig, description="Distribution analysis configuration" - ) - - -# ============================================================================= -# Module B: Cross-Feature Analysis -# ============================================================================= - - -class CorrelationConfig(BaseConfig): - """Configuration for correlation analysis. - - Analyzes relationships between features to detect: - - Linear relationships (Pearson) - - Monotonic relationships (Spearman) - - General dependence (Kendall) - - Lagged relationships - - Attributes: - enabled: Run correlation analysis - methods: Correlation methods to use - compute_pairwise: Compute all pairwise correlations (vs just with outcome) - min_periods: Minimum observations for correlation - lag_correlations: Compute lagged cross-correlations - max_lag: Maximum lag for cross-correlations - - Examples: - >>> # Default: Pearson only - >>> config = CorrelationConfig() - - >>> # Custom: Multiple methods with lags - >>> config = CorrelationConfig( - ... methods=[CorrelationMethod.PEARSON, CorrelationMethod.SPEARMAN], - ... lag_correlations=True, - ... max_lag=10 - ... ) - """ - - enabled: bool = Field(True, description="Run correlation analysis") - methods: list[CorrelationMethod] = Field( - default_factory=lambda: [CorrelationMethod.PEARSON], - description="Correlation methods: pearson, spearman, kendall", - ) - compute_pairwise: bool = Field( - True, description="Compute all pairwise correlations (vs just with outcome)" - ) - min_periods: PositiveInt = Field(30, description="Minimum observations for correlation") - lag_correlations: bool = Field(False, description="Compute lagged cross-correlations") - max_lag: PositiveInt = Field(10, description="Maximum lag for cross-correlations") - - @field_validator("methods") - @classmethod - def check_methods(cls, v: list[CorrelationMethod]) -> list[CorrelationMethod]: - """Ensure at least one method specified.""" - if not v: - raise ValueError("Must specify at least one correlation method") - return v - - -class PCAConfig(BaseConfig): - """Configuration for Principal Component Analysis (PCA). - - Dimensionality reduction and feature redundancy analysis: - - Identify principal components - - Measure explained variance - - Detect redundancy - - Attributes: - enabled: Run PCA - n_components: Number of components (int, float for variance %, or "auto") - variance_threshold: Cumulative variance to explain (for n_components="auto") - standardize: Standardize features before PCA (recommended) - rotation: Optional rotation for interpretability ("varimax", "quartimax") - - Examples: - >>> # Default: Disabled (opt-in) - >>> config = PCAConfig() - - >>> # Custom: Explain 95% of variance - >>> config = PCAConfig( - ... enabled=True, - ... n_components="auto", - ... variance_threshold=0.95 - ... ) - - >>> # Custom: Exactly 5 components - >>> config = PCAConfig(enabled=True, n_components=5) - """ - - enabled: bool = Field(False, description="Run PCA (opt-in, can be expensive)") - n_components: PositiveInt | Probability | Literal["auto"] = Field( - "auto", description="Number of components: int (exact), float (variance %), or auto" - ) - variance_threshold: Probability = Field( - 0.95, description="Cumulative variance to explain (for n_components='auto')" - ) - standardize: bool = Field( - True, description="Standardize features before PCA (strongly recommended)" - ) - rotation: Literal["varimax", "quartimax"] | None = Field( - None, description="Optional rotation for interpretability" - ) - - @model_validator(mode="after") - def check_n_components_config(self) -> PCAConfig: - """Validate n_components configuration.""" - if not self.enabled: - return self - - if self.n_components == "auto" and not (0 < self.variance_threshold < 1): - raise ValueError("variance_threshold must be in (0, 1) when n_components='auto'") - - return self - - -class ClusteringConfig(BaseConfig): - """Configuration for feature clustering. - - Groups similar features to detect: - - Redundant features - - Feature families - - Latent structure - - Attributes: - enabled: Run clustering - method: Clustering algorithm - n_clusters: Number of clusters (int or "auto") - linkage: Linkage method for hierarchical clustering - distance_metric: Distance metric - min_cluster_size: Minimum cluster size (for DBSCAN) - eps: DBSCAN epsilon parameter - - Examples: - >>> # Default: Disabled (opt-in) - >>> config = ClusteringConfig() - - >>> # Custom: Hierarchical with auto clusters - >>> config = ClusteringConfig( - ... enabled=True, - ... method=ClusteringMethod.HIERARCHICAL, - ... n_clusters="auto", - ... linkage=LinkageMethod.WARD - ... ) - """ - - enabled: bool = Field(False, description="Run clustering (opt-in)") - method: ClusteringMethod = Field(ClusteringMethod.HIERARCHICAL, description="Clustering method") - n_clusters: PositiveInt | Literal["auto"] = Field( - "auto", description="Number of clusters (auto uses elbow method)" - ) - linkage: LinkageMethod = Field( - LinkageMethod.WARD, description="Linkage method for hierarchical clustering" - ) - distance_metric: DistanceMetric = Field(DistanceMetric.EUCLIDEAN, description="Distance metric") - min_cluster_size: PositiveInt = Field(5, description="Minimum cluster size (DBSCAN)") - eps: PositiveFloat = Field(0.5, description="DBSCAN epsilon parameter") - - -class RedundancyConfig(BaseConfig): - """Configuration for feature redundancy detection. - - Identifies redundant features using: - - Pairwise correlation thresholds - - Variance Inflation Factor (VIF) - - Attributes: - enabled: Run redundancy detection - correlation_threshold: Correlation threshold for redundancy - compute_vif: Compute Variance Inflation Factor - vif_threshold: VIF threshold for multicollinearity - keep_strategy: Which feature to keep when redundant ("first", "last", "highest_ic") - - Examples: - >>> # Default: correlation > 0.95 - >>> config = RedundancyConfig() - - >>> # Custom: VIF-based with 0.90 threshold - >>> config = RedundancyConfig( - ... correlation_threshold=0.90, - ... compute_vif=True, - ... vif_threshold=5.0 - ... ) - """ - - enabled: bool = Field(True, description="Run redundancy detection") - correlation_threshold: Probability = Field( - 0.95, description="Correlation threshold for redundancy" - ) - compute_vif: bool = Field(False, description="Compute VIF (can be slow for many features)") - vif_threshold: PositiveFloat = Field(10.0, description="VIF threshold for multicollinearity") - keep_strategy: Literal["first", "last", "highest_ic"] = Field( - "highest_ic", description="Which feature to keep when redundant" - ) - - -class ModuleBConfig(BaseConfig): - """Configuration for Module B: Cross-Feature Analysis. - - Analyzes relationships between features: - - Correlation (linear, monotonic, lagged) - - PCA (dimensionality reduction) - - Clustering (feature grouping) - - Redundancy (multicollinearity) - - Examples: - >>> # Default: Correlation + redundancy only - >>> config = ModuleBConfig() - - >>> # Custom: Full analysis with PCA - >>> config = ModuleBConfig( - ... correlation=CorrelationConfig(lag_correlations=True), - ... pca=PCAConfig(enabled=True), - ... clustering=ClusteringConfig(enabled=True) - ... ) - """ - - correlation: CorrelationConfig = Field( - default_factory=CorrelationConfig, description="Correlation analysis configuration" - ) - pca: PCAConfig = Field(default_factory=PCAConfig, description="PCA configuration") - clustering: ClusteringConfig = Field( - default_factory=ClusteringConfig, description="Clustering configuration" - ) - redundancy: RedundancyConfig = Field( - default_factory=RedundancyConfig, description="Redundancy detection configuration" - ) - - -# ============================================================================= -# Module C: Feature-Outcome Relationships -# ============================================================================= - - -class ICConfig(BaseConfig): - """Configuration for Information Coefficient (IC) analysis. - - Measures predictive power of features: - - Contemporaneous IC (lag 0) - - Forward-looking IC (lag > 0) - - HAC-adjusted IC (autocorrelation correction) - - IC decay over time - - Attributes: - enabled: Run IC analysis - method: Correlation method for IC - lag_structure: Lags to analyze (e.g., [0, 1, 5, 10, 21]) - hac_adjustment: Apply Newey-West HAC adjustment - max_lag_hac: Maximum lag for HAC (auto = int(4*(n/100)^(2/9))) - compute_t_stats: Compute t-statistics for IC - compute_decay: Analyze IC decay over time - - Examples: - >>> # Default: Pearson IC at lags 0, 1, 5 - >>> config = ICConfig() - - >>> # Custom: Spearman IC with HAC adjustment - >>> config = ICConfig( - ... method=CorrelationMethod.SPEARMAN, - ... lag_structure=[0, 1, 5, 10, 21], - ... hac_adjustment=True - ... ) - - References: - - Newey, W.K. and West, K.D. (1987). "A Simple, Positive Semi-Definite, - Heteroskedasticity and Autocorrelation Consistent Covariance Matrix." - """ - - enabled: bool = Field(True, description="Run IC analysis") - method: CorrelationMethod = Field( - CorrelationMethod.PEARSON, description="Correlation method for IC" - ) - lag_structure: list[NonNegativeInt] = Field( - default_factory=lambda: [0, 1, 5], description="Lags to analyze forward returns" - ) - hac_adjustment: bool = Field(False, description="Apply Newey-West HAC adjustment (expensive)") - max_lag_hac: PositiveInt | Literal["auto"] = Field( - "auto", description="Maximum lag for HAC adjustment" - ) - compute_t_stats: bool = Field(True, description="Compute t-statistics for IC") - compute_decay: bool = Field(False, description="Analyze IC decay over time (expensive)") - - @field_validator("lag_structure") - @classmethod - def check_lag_structure(cls, v: list[int]) -> list[int]: - """Ensure lag structure is valid.""" - if not v: - raise ValueError("Must specify at least one lag") - if any(lag < 0 for lag in v): - raise ValueError("Lags must be non-negative") - return sorted(v) - - -class BinaryClassificationConfig(BaseConfig): - """Configuration for binary classification metrics. - - Evaluates signals as binary predictions: - - Precision: % of predicted positives that are correct - - Recall: % of actual positives that are detected - - F1: Harmonic mean of precision and recall - - Lift: Improvement over random - - Coverage: % of universe with signals - - Attributes: - enabled: Run binary classification analysis - thresholds: Thresholds for converting scores to binary predictions - metrics: Metrics to compute - positive_class: What constitutes a "positive" signal - compute_confusion_matrix: Compute confusion matrix - compute_roc_curve: Compute ROC curve and AUC - - Examples: - >>> # Default: Disabled (requires threshold selection) - >>> config = BinaryClassificationConfig() - - >>> # Custom: Multiple thresholds - >>> config = BinaryClassificationConfig( - ... enabled=True, - ... thresholds=[0.0, 0.5, 1.0], - ... metrics=["precision", "recall", "f1", "lift"] - ... ) - """ - - enabled: bool = Field(False, description="Run binary classification analysis (opt-in)") - thresholds: list[float] = Field( - default_factory=lambda: [0.0], description="Thresholds for binary conversion" - ) - metrics: list[Literal["precision", "recall", "f1", "lift", "coverage"]] = Field( - default_factory=lambda: ["precision", "recall", "f1"], - description="Metrics to compute", - ) - positive_class: int | str = Field(1, description="Positive class label") - compute_confusion_matrix: bool = Field(True, description="Compute confusion matrix") - compute_roc_curve: bool = Field(False, description="Compute ROC curve (expensive)") - - -class ThresholdAnalysisConfig(BaseConfig): - """Configuration for threshold optimization and sensitivity analysis. - - Sweeps thresholds to find optimal values: - - Maximize Sharpe, precision, recall, etc. - - Subject to constraints (e.g., coverage >= 30%) - - Sensitivity analysis - - Attributes: - enabled: Run threshold analysis - sweep_range: (min, max) threshold range to sweep - n_points: Number of points in sweep - optimization_target: What to optimize - constraint_metric: Optional constraint metric - constraint_value: Constraint threshold - constraint_type: Constraint type (">=", "<=", "==") - - Examples: - >>> # Default: Disabled (expensive) - >>> config = ThresholdAnalysisConfig() - - >>> # Custom: Maximize Sharpe with 30% coverage - >>> config = ThresholdAnalysisConfig( - ... enabled=True, - ... sweep_range=(-2.0, 2.0), - ... n_points=100, - ... optimization_target=ThresholdOptimizationTarget.SHARPE, - ... constraint_metric="coverage", - ... constraint_value=0.30, - ... constraint_type=">=" - ... ) - """ - - enabled: bool = Field(False, description="Run threshold analysis (expensive, opt-in)") - sweep_range: tuple[float, float] = Field((-2.0, 2.0), description="(min, max) threshold range") - n_points: PositiveInt = Field(50, description="Number of points in sweep") - optimization_target: ThresholdOptimizationTarget = Field( - ThresholdOptimizationTarget.SHARPE, description="Optimization objective" - ) - constraint_metric: str | None = Field(None, description="Optional constraint metric") - constraint_value: float | None = Field(None, description="Constraint threshold") - constraint_type: Literal[">=", "<=", "=="] = Field(">=", description="Constraint type") - - @model_validator(mode="after") - def validate_sweep_range(self) -> ThresholdAnalysisConfig: - """Validate sweep range.""" - if self.enabled: - validate_min_max_range(self.sweep_range[0], self.sweep_range[1], "sweep_range") - return self - - @model_validator(mode="after") - def validate_constraint(self) -> ThresholdAnalysisConfig: - """Validate constraint configuration.""" - has_metric = self.constraint_metric is not None - has_value = self.constraint_value is not None - - if has_metric != has_value: - raise ValueError( - "Both constraint_metric and constraint_value must be set (or both None)" - ) - - return self - - -class MLDiagnosticsConfig(BaseConfig): - """Configuration for ML model diagnostics. - - Advanced feature analysis using ML: - - Feature importance (tree-based, permutation) - - SHAP values (Shapley Additive Explanations) - - Feature drift detection - - Interaction detection - - Attributes: - enabled: Run ML diagnostics - feature_importance: Compute feature importance - importance_method: Importance method ("tree", "permutation") - shap_analysis: Compute SHAP values (very expensive) - shap_sample_size: Subsample size for SHAP (None for full) - drift_detection: Detect feature drift over time - drift_method: Drift detection method - drift_window: Rolling window for drift detection - - Examples: - >>> # Default: Feature importance only - >>> config = MLDiagnosticsConfig() - - >>> # Custom: Full analysis with SHAP - >>> config = MLDiagnosticsConfig( - ... shap_analysis=True, - ... shap_sample_size=1000, - ... drift_detection=True - ... ) - - Warning: - SHAP analysis can be very slow for large datasets. Use shap_sample_size - to limit computation time. - """ - - enabled: bool = Field(True, description="Run ML diagnostics") - feature_importance: bool = Field(True, description="Compute feature importance") - importance_method: Literal["tree", "permutation"] = Field( - "tree", description="Importance method: tree-based or permutation" - ) - shap_analysis: bool = Field(False, description="Compute SHAP values (very expensive)") - shap_sample_size: PositiveInt | None = Field( - None, description="Subsample for SHAP (None = all data)" - ) - drift_detection: bool = Field(False, description="Detect feature drift over time") - drift_method: DriftDetectionMethod = Field( - DriftDetectionMethod.KOLMOGOROV_SMIRNOV, description="Drift detection method" - ) - drift_window: PositiveInt = Field(63, description="Rolling window for drift detection (days)") - - -class ModuleCConfig(BaseConfig): - """Configuration for Module C: Feature-Outcome Relationships. - - Analyzes how features relate to outcomes: - - IC analysis (predictive power) - - Binary classification (signal quality) - - Threshold optimization - - ML diagnostics (importance, drift) - - Examples: - >>> # Default: IC + ML diagnostics - >>> config = ModuleCConfig() - - >>> # Custom: Full analysis - >>> config = ModuleCConfig( - ... ic=ICConfig(lag_structure=[0, 1, 5, 10, 21]), - ... binary_classification=BinaryClassificationConfig(enabled=True), - ... threshold_analysis=ThresholdAnalysisConfig(enabled=True), - ... ml_diagnostics=MLDiagnosticsConfig(shap_analysis=True) - ... ) - """ - - ic: ICConfig = Field(default_factory=ICConfig, description="IC analysis configuration") - binary_classification: BinaryClassificationConfig = Field( - default_factory=BinaryClassificationConfig, - description="Binary classification configuration", - ) - threshold_analysis: ThresholdAnalysisConfig = Field( - default_factory=ThresholdAnalysisConfig, description="Threshold analysis configuration" - ) - ml_diagnostics: MLDiagnosticsConfig = Field( - default_factory=MLDiagnosticsConfig, description="ML diagnostics configuration" - ) - - -# ============================================================================= -# Top-Level Feature Evaluator Configuration -# ============================================================================= - - -class FeatureEvaluatorConfig(BaseConfig): - """Top-level configuration for feature evaluation (Modules A, B, C). - - Orchestrates comprehensive feature analysis: - - **Module A**: Individual feature diagnostics - - **Module B**: Cross-feature relationships - - **Module C**: Feature-outcome relationships - - Attributes: - module_a: Feature diagnostics configuration - module_b: Cross-feature analysis configuration - module_c: Feature-outcome configuration - export_recommendations: Export preprocessing recommendations - export_to_qfeatures: Export in qfeatures-compatible format - return_dataframes: Return metrics as DataFrames - n_jobs: Parallel processing (-1 for all cores) - cache_enabled: Enable caching of expensive computations - cache_dir: Cache directory - verbose: Enable verbose output - - Examples: - >>> # Quick start with defaults - >>> config = FeatureEvaluatorConfig() - >>> evaluator = FeatureEvaluator(config) - >>> results = evaluator.evaluate(features_df, outcomes_df) - - >>> # Load from YAML - >>> config = FeatureEvaluatorConfig.from_yaml("feature_config.yaml") - - >>> # Use preset - >>> config = FeatureEvaluatorConfig.for_quick_analysis() - - >>> # Custom configuration - >>> config = FeatureEvaluatorConfig( - ... module_a=ModuleAConfig( - ... stationarity=StationarityConfig(significance_level=0.01) - ... ), - ... module_c=ModuleCConfig( - ... ic=ICConfig(lag_structure=[0, 1, 5, 10, 21]) - ... ), - ... n_jobs=-1 - ... ) - """ - - module_a: ModuleAConfig = Field( - default_factory=ModuleAConfig, description="Feature diagnostics (Module A)" - ) - module_b: ModuleBConfig = Field( - default_factory=ModuleBConfig, description="Cross-feature analysis (Module B)" - ) - module_c: ModuleCConfig = Field( - default_factory=ModuleCConfig, description="Feature-outcome analysis (Module C)" - ) - - # Integration settings - export_recommendations: bool = Field(True, description="Export preprocessing recommendations") - export_to_qfeatures: bool = Field(False, description="Export in qfeatures-compatible format") - return_dataframes: bool = Field(True, description="Return metrics as DataFrames") - - # Computational settings - n_jobs: int = Field(-1, ge=-1, description="Parallel jobs (-1 = all cores)") - cache_enabled: bool = Field(True, description="Enable caching") - cache_dir: Path = Field( - default_factory=lambda: Path.home() / ".cache" / "qeval" / "features", - description="Cache directory", - ) - verbose: bool = Field(False, description="Verbose output") - - @classmethod - def for_quick_analysis(cls) -> FeatureEvaluatorConfig: - """Preset for quick exploratory analysis (fast, essential diagnostics only). - - Returns: - Config optimized for speed - """ - return cls( - module_a=ModuleAConfig( - stationarity=StationarityConfig(pp_enabled=False), - volatility=VolatilityConfig(detect_clustering=False), - distribution=DistributionConfig(detect_outliers=False), - ), - module_b=ModuleBConfig( - correlation=CorrelationConfig(lag_correlations=False), - pca=PCAConfig(enabled=False), - clustering=ClusteringConfig(enabled=False), - ), - module_c=ModuleCConfig( - ic=ICConfig(hac_adjustment=False, compute_decay=False), - ml_diagnostics=MLDiagnosticsConfig(shap_analysis=False, drift_detection=False), - ), - ) - - @classmethod - def for_research(cls) -> FeatureEvaluatorConfig: - """Preset for academic research (comprehensive, expensive analyses enabled). - - Returns: - Config with all analyses enabled - """ - return cls( - module_a=ModuleAConfig( - stationarity=StationarityConfig(pp_enabled=True), - volatility=VolatilityConfig(window_sizes=[10, 21, 63]), - distribution=DistributionConfig( - detect_outliers=True, - normality_tests=[ - NormalityTest.JARQUE_BERA, - NormalityTest.SHAPIRO, - NormalityTest.ANDERSON, - ], - ), - ), - module_b=ModuleBConfig( - correlation=CorrelationConfig( - methods=[ - CorrelationMethod.PEARSON, - CorrelationMethod.SPEARMAN, - CorrelationMethod.KENDALL, - ], - lag_correlations=True, - ), - pca=PCAConfig(enabled=True), - clustering=ClusteringConfig(enabled=True), - ), - module_c=ModuleCConfig( - ic=ICConfig( - lag_structure=[0, 1, 5, 10, 21], - hac_adjustment=True, - compute_decay=True, - ), - binary_classification=BinaryClassificationConfig(enabled=True), - threshold_analysis=ThresholdAnalysisConfig(enabled=True), - ml_diagnostics=MLDiagnosticsConfig( - shap_analysis=True, - drift_detection=True, - ), - ), - ) - - @classmethod - def for_production(cls) -> FeatureEvaluatorConfig: - """Preset for production monitoring (fast, focused on drift and degradation). - - Returns: - Config optimized for production monitoring - """ - return cls( - module_a=ModuleAConfig( - stationarity=StationarityConfig(pp_enabled=False), - acf=ACFConfig(enabled=False), - volatility=VolatilityConfig(enabled=False), - distribution=DistributionConfig(test_normality=False, compute_moments=True), - ), - module_b=ModuleBConfig( - correlation=CorrelationConfig(lag_correlations=False), - pca=PCAConfig(enabled=False), - clustering=ClusteringConfig(enabled=False), - ), - module_c=ModuleCConfig( - ic=ICConfig(compute_decay=False), - ml_diagnostics=MLDiagnosticsConfig( - feature_importance=True, - drift_detection=True, - drift_window=21, # Faster detection - ), - ), - ) diff --git a/src/ml4t/engineer/config/labeling.py b/src/ml4t/engineer/config/labeling.py index 78fd26d..21f1e57 100644 --- a/src/ml4t/engineer/config/labeling.py +++ b/src/ml4t/engineer/config/labeling.py @@ -451,16 +451,3 @@ def trend_scanning( __all__ = [ "LabelingConfig", ] - -_REMOVED_EXPORTS = { - "BarrierLabelingConfig": ( - "ml4t.engineer.config.labeling.BarrierLabelingConfig has been removed. " - "Use ml4t.engineer.config.LabelingConfig instead." - ) -} - - -def __getattr__(name: str) -> object: - if name in _REMOVED_EXPORTS: - raise ImportError(_REMOVED_EXPORTS[name]) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ml4t/engineer/config/validation.py b/src/ml4t/engineer/config/validation.py deleted file mode 100644 index e536853..0000000 --- a/src/ml4t/engineer/config/validation.py +++ /dev/null @@ -1,281 +0,0 @@ -# ruff: noqa: UP006, UP045 -"""Custom validators and validation utilities. - -This module provides reusable validators, custom types, and validation -helpers used across the configuration system. -""" - -from __future__ import annotations - -from enum import Enum -from typing import Annotated - -from pydantic import Field - -# Custom type aliases for common constraints -PositiveInt = Annotated[int, Field(gt=0)] -NonNegativeInt = Annotated[int, Field(ge=0)] -PositiveFloat = Annotated[float, Field(gt=0.0)] -NonNegativeFloat = Annotated[float, Field(ge=0.0)] -Probability = Annotated[float, Field(ge=0.0, le=1.0)] -CorrelationValue = Annotated[float, Field(ge=-1.0, le=1.0)] - - -class SignificanceLevel(float, Enum): - """Standard significance levels for hypothesis testing.""" - - LEVEL_01 = 0.01 - LEVEL_05 = 0.05 - LEVEL_10 = 0.10 - - -class CorrelationMethod(str, Enum): - """Correlation calculation methods.""" - - PEARSON = "pearson" - SPEARMAN = "spearman" - KENDALL = "kendall" - - -class StationarityTest(str, Enum): - """Stationarity test types.""" - - ADF = "adf" # Augmented Dickey-Fuller - KPSS = "kpss" # Kwiatkowski-Phillips-Schmidt-Shin - PP = "pp" # Phillips-Perron - - -class RegressionType(str, Enum): - """Regression types for stationarity tests.""" - - CONSTANT = "c" # Constant only - CONSTANT_TREND = "ct" # Constant and trend - CONSTANT_TREND_SQUARED = "ctt" # Constant, trend, and trend squared - NONE = "n" # No constant or trend - - -class ClusteringMethod(str, Enum): - """Clustering algorithm types.""" - - HIERARCHICAL = "hierarchical" - KMEANS = "kmeans" - DBSCAN = "dbscan" - - -class LinkageMethod(str, Enum): - """Linkage methods for hierarchical clustering.""" - - WARD = "ward" - COMPLETE = "complete" - AVERAGE = "average" - SINGLE = "single" - - -class DistanceMetric(str, Enum): - """Distance metrics for clustering.""" - - EUCLIDEAN = "euclidean" - CORRELATION = "correlation" - MANHATTAN = "manhattan" - COSINE = "cosine" - - -class NormalityTest(str, Enum): - """Normality test types.""" - - JARQUE_BERA = "jarque_bera" - SHAPIRO = "shapiro" - KOLMOGOROV_SMIRNOV = "ks" - ANDERSON = "anderson" - - -class OutlierMethod(str, Enum): - """Outlier detection methods.""" - - ZSCORE = "zscore" - IQR = "iqr" - ISOLATION_FOREST = "isolation_forest" - - -class VolatilityClusterMethod(str, Enum): - """Methods for detecting volatility clustering.""" - - LJUNG_BOX = "ljung_box" - ENGLE_ARCH = "engle_arch" - - -class ThresholdOptimizationTarget(str, Enum): - """Optimization targets for threshold analysis.""" - - SHARPE = "sharpe" - PRECISION = "precision" - RECALL = "recall" - F1 = "f1" - INFORMATION_COEFFICIENT = "ic" - - -class DriftDetectionMethod(str, Enum): - """Feature drift detection methods.""" - - KOLMOGOROV_SMIRNOV = "ks" - WASSERSTEIN = "wasserstein" - PSI = "psi" # Population Stability Index - - -class PortfolioMetric(str, Enum): - """Portfolio performance metrics.""" - - SHARPE = "sharpe" - SORTINO = "sortino" - CALMAR = "calmar" - MAX_DRAWDOWN = "max_dd" - VAR = "var" # Value at Risk - CVAR = "cvar" # Conditional Value at Risk - OMEGA = "omega" - - -class TimeFrequency(str, Enum): - """Time aggregation frequencies.""" - - DAILY = "daily" - WEEKLY = "weekly" - MONTHLY = "monthly" - QUARTERLY = "quarterly" - ANNUAL = "annual" - - -class FDRMethod(str, Enum): - """False Discovery Rate control methods.""" - - BONFERRONI = "bonferroni" - HOLM = "holm" - BENJAMINI_HOCHBERG = "bh" - BENJAMINI_YEKUTIELI = "by" - - -class BayesianPriorDistribution(str, Enum): - """Prior distributions for Bayesian analysis.""" - - NORMAL = "normal" - STUDENT_T = "student_t" - UNIFORM = "uniform" - - -class ReportFormat(str, Enum): - """Report output formats.""" - - HTML = "html" - JSON = "json" - PDF = "pdf" - - -class ReportTemplate(str, Enum): - """Report templates.""" - - FULL = "full" - SUMMARY = "summary" - DIAGNOSTIC = "diagnostic" - - -class ReportTheme(str, Enum): - """Report visual themes.""" - - LIGHT = "light" - DARK = "dark" - PROFESSIONAL = "professional" - - -class TableFormat(str, Enum): - """Table formatting styles.""" - - STYLED = "styled" - PLAIN = "plain" - DATATABLES = "datatables" - - -class DataFrameExportFormat(str, Enum): - """DataFrame serialization formats for JSON.""" - - RECORDS = "records" # list of dicts - SPLIT = "split" # {index: [...], columns: [...], data: [...]} - INDEX = "index" # {index: {column: value}} - - -def validate_positive_int(v: int, field_name: str = "value") -> int: - """Validate that an integer is positive. - - Args: - v: Value to validate - field_name: Name of field for error messages - - Returns: - Validated value - - Raises: - ValueError: If value is not positive - """ - if v <= 0: - raise ValueError(f"{field_name} must be positive (got {v})") - return v - - -def validate_probability(v: float, field_name: str = "probability") -> float: - """Validate that a float is in [0, 1]. - - Args: - v: Value to validate - field_name: Name of field for error messages - - Returns: - Validated value - - Raises: - ValueError: If value is not in [0, 1] - """ - if not 0.0 <= v <= 1.0: - raise ValueError(f"{field_name} must be in [0, 1] (got {v})") - return v - - -def validate_significance_level(v: float) -> float: - """Validate significance level is a standard value. - - Args: - v: Significance level - - Returns: - Validated significance level - - Raises: - ValueError: If not a standard significance level - """ - standard_levels = {0.01, 0.05, 0.10} - if v not in standard_levels: - raise ValueError( - f"Significance level {v} is non-standard. " - f"Consider using 0.01, 0.05, or 0.10 for interpretability." - ) - return v - - -def validate_min_max_range( - min_val: float, max_val: float, field_prefix: str = "range" -) -> tuple[float, float]: - """Validate that min < max. - - Args: - min_val: Minimum value - max_val: Maximum value - field_prefix: Prefix for error messages - - Returns: - Validated (min, max) tuple - - Raises: - ValueError: If min >= max - """ - if min_val >= max_val: - raise ValueError( - f"{field_prefix}_min must be < {field_prefix}_max (got {min_val} >= {max_val})" - ) - return min_val, max_val diff --git a/src/ml4t/engineer/core/calendars/equity.py b/src/ml4t/engineer/core/calendars/equity.py index daa6cfb..e98bfb3 100644 --- a/src/ml4t/engineer/core/calendars/equity.py +++ b/src/ml4t/engineer/core/calendars/equity.py @@ -120,17 +120,18 @@ def _next_basic_open(self, dt: datetime) -> datetime: # Ensure we're working in market timezone dt = dt.replace(tzinfo=self._tz) if dt.tzinfo is None else dt.astimezone(self._tz) - # Start from next day if after market close - if dt.time() >= time(16, 0): - dt = dt.replace(hour=9, minute=30, second=0, microsecond=0) + timedelta(days=1) + # If before today's open, next open is today 9:30 + if dt.time() < time(9, 30): + candidate = dt.replace(hour=9, minute=30, second=0, microsecond=0) else: - dt = dt.replace(hour=9, minute=30, second=0, microsecond=0) + # During or after market hours — next open is tomorrow 9:30 + candidate = dt.replace(hour=9, minute=30, second=0, microsecond=0) + timedelta(days=1) # Skip weekends - while dt.weekday() >= 5: - dt += timedelta(days=1) + while candidate.weekday() >= 5: + candidate += timedelta(days=1) - return dt + return candidate def previous_close(self, dt: datetime) -> datetime: """Previous market close before given datetime.""" @@ -157,17 +158,18 @@ def _previous_basic_close(self, dt: datetime) -> datetime: # Ensure we're working in market timezone dt = dt.replace(tzinfo=self._tz) if dt.tzinfo is None else dt.astimezone(self._tz) - # If before market open, go to previous day - if dt.time() < time(9, 30): - dt = dt.replace(hour=16, minute=0, second=0, microsecond=0) - timedelta(days=1) + # If after today's close, previous close is today 16:00 + if dt.time() > time(16, 0): + candidate = dt.replace(hour=16, minute=0, second=0, microsecond=0) else: - dt = dt.replace(hour=16, minute=0, second=0, microsecond=0) + # Before or during market hours — previous close is yesterday 16:00 + candidate = dt.replace(hour=16, minute=0, second=0, microsecond=0) - timedelta(days=1) # Skip weekends backwards - while dt.weekday() >= 5: - dt -= timedelta(days=1) + while candidate.weekday() >= 5: + candidate -= timedelta(days=1) - return dt + return candidate def sessions_between(self, start: datetime, end: datetime) -> list[datetime]: """List all trading sessions between dates.""" diff --git a/src/ml4t/engineer/core/deprecation.py b/src/ml4t/engineer/core/deprecation.py deleted file mode 100644 index ce7fefd..0000000 --- a/src/ml4t/engineer/core/deprecation.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Parameter deprecation utilities for ML4T Engineer. - -Provides utilities for deprecating parameters with clear migration paths. -""" - -from __future__ import annotations - -import warnings -from typing import TypeVar - -T = TypeVar("T") - - -def resolve_period_parameter( - *, - period: int | None, - timeperiod: int | None = None, - window: int | None = None, - lookback: int | None = None, - default: int, - func_name: str = "", -) -> int: - """ - Resolve period parameter from multiple possible sources. - - This function handles backward compatibility for the standardization - from `timeperiod`, `window`, and `lookback` to `period`. - - Parameters - ---------- - period : int | None - The preferred parameter name (new standard) - timeperiod : int | None - Deprecated TA-Lib style parameter - window : int | None - Deprecated rolling window style parameter - lookback : int | None - Deprecated lookback style parameter - default : int - Default value if none provided - func_name : str - Name of the function for warning messages - - Returns - ------- - int - The resolved period value - - Raises - ------ - ValueError - If multiple conflicting values are provided - - Examples - -------- - >>> period = resolve_period_parameter( - ... period=None, timeperiod=14, default=10, func_name="rsi" - ... ) - >>> # Issues deprecation warning and returns 14 - """ - # Collect all provided values - provided: dict[str, int] = {} - deprecated_params: dict[str, int] = {} - - if period is not None: - provided["period"] = period - if timeperiod is not None: - deprecated_params["timeperiod"] = timeperiod - if window is not None: - deprecated_params["window"] = window - if lookback is not None: - deprecated_params["lookback"] = lookback - - # If period is provided, use it (ignore deprecated) - if period is not None: - if deprecated_params: - # Warn about redundant deprecated params - deprecated_names = ", ".join(f"'{k}'" for k in deprecated_params) - warnings.warn( - f"Both 'period' and deprecated parameter(s) {deprecated_names} provided " - f"to {func_name}. Using 'period={period}'.", - DeprecationWarning, - stacklevel=3, - ) - return period - - # Check deprecated parameters - if deprecated_params: - if len(deprecated_params) > 1: - raise ValueError( - f"Multiple deprecated period parameters provided to {func_name}: " - f"{list(deprecated_params.keys())}. Use 'period' instead." - ) - - # Get the single deprecated param - param_name, param_value = next(iter(deprecated_params.items())) - warnings.warn( - f"'{param_name}' parameter is deprecated, use 'period' instead in {func_name}.", - DeprecationWarning, - stacklevel=3, - ) - return param_value - - # No period provided, use default - return default - - -def deprecated_parameter( - old_name: str, - new_name: str = "period", - *, - func_name: str = "", -) -> None: - """ - Issue a deprecation warning for a renamed parameter. - - Parameters - ---------- - old_name : str - The deprecated parameter name - new_name : str - The new parameter name (default: "period") - func_name : str - Name of the function for the warning message - """ - warnings.warn( - f"'{old_name}' parameter is deprecated, use '{new_name}' instead" - + (f" in {func_name}" if func_name else "") - + ".", - DeprecationWarning, - stacklevel=3, - ) - - -__all__ = ["resolve_period_parameter", "deprecated_parameter"] diff --git a/src/ml4t/engineer/features/momentum/mom.py b/src/ml4t/engineer/features/momentum/mom.py index cd9f127..fbbda2f 100644 --- a/src/ml4t/engineer/features/momentum/mom.py +++ b/src/ml4t/engineer/features/momentum/mom.py @@ -13,7 +13,6 @@ from numba import jit from ml4t.engineer.core.decorators import feature -from ml4t.engineer.core.deprecation import resolve_period_parameter from ml4t.engineer.core.exceptions import InvalidParameterError @@ -83,8 +82,6 @@ def mom_polars(column: str, period: int = 10) -> pl.Expr: def mom( close: npt.NDArray[np.float64] | pl.Series | str, period: int | None = None, - *, - timeperiod: int | None = None, # Deprecated alias for period ) -> npt.NDArray[np.float64] | pl.Expr: """ Momentum exactly matching TA-Lib. @@ -99,8 +96,6 @@ def mom( Price data or column name period : int, default 10 Number of periods for momentum calculation - timeperiod : int, optional - Deprecated alias for period. Use period instead. Returns ------- @@ -120,13 +115,8 @@ def mom( - First 'period' close will be NaN - Simple calculation: price - price[n periods ago] """ - # Resolve period with deprecation handling - period = resolve_period_parameter( - period=period, - timeperiod=timeperiod, - default=10, - func_name="mom", - ) + if period is None: + period = 10 # Validate parameters if period < 1: diff --git a/src/ml4t/engineer/labeling/__init__.py b/src/ml4t/engineer/labeling/__init__.py index 2c56041..49e9b1e 100644 --- a/src/ml4t/engineer/labeling/__init__.py +++ b/src/ml4t/engineer/labeling/__init__.py @@ -92,13 +92,6 @@ trend_scanning_feature, ] -_REMOVED_EXPORTS = { - "BarrierConfig": ( - "ml4t.engineer.labeling.BarrierConfig has been removed. " - "Use ml4t.engineer.config.LabelingConfig.triple_barrier(...) instead." - ) -} - def register_labeling_features(registry: object = None) -> int: """ @@ -128,9 +121,3 @@ def register_labeling_features(registry: object = None) -> int: registry.register(feature) # type: ignore[attr-defined] return len(ALL_LABELING_FEATURES) - - -def __getattr__(name: str) -> object: - if name in _REMOVED_EXPORTS: - raise ImportError(_REMOVED_EXPORTS[name]) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ml4t/engineer/labeling/barrier_utils.py b/src/ml4t/engineer/labeling/barrier_utils.py deleted file mode 100644 index c9c94d7..0000000 --- a/src/ml4t/engineer/labeling/barrier_utils.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Compatibility shim for removed legacy barrier utility module.""" - -_REMOVAL_MESSAGE = ( - "ml4t.engineer.labeling.barrier_utils has been removed. " - "Use ml4t.engineer.labeling.triple_barrier.triple_barrier_labels for labeling. " - "Legacy helpers apply_triple_barrier, calculate_returns, and compute_barrier_touches " - "are no longer supported." -) - -raise ImportError(_REMOVAL_MESSAGE) diff --git a/src/ml4t/engineer/labeling/barriers.py b/src/ml4t/engineer/labeling/barriers.py deleted file mode 100644 index 43be592..0000000 --- a/src/ml4t/engineer/labeling/barriers.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility shim for removed legacy barrier config module.""" - -_REMOVAL_MESSAGE = ( - "ml4t.engineer.labeling.barriers has been removed. " - "Use ml4t.engineer.config.LabelingConfig.triple_barrier(...) for triple-barrier settings " - "or LabelingConfig.atr_barrier(...) for ATR-adjusted settings." -) - -raise ImportError(_REMOVAL_MESSAGE) diff --git a/src/ml4t/engineer/labeling/core.py b/src/ml4t/engineer/labeling/core.py deleted file mode 100644 index bee28cf..0000000 --- a/src/ml4t/engineer/labeling/core.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Compatibility shim for removed labeling.core module.""" - -_REMOVAL_MESSAGE = ( - "ml4t.engineer.labeling.core has been removed. " - "Use ml4t.engineer.labeling.triple_barrier for triple_barrier_labels " - "(with ml4t.engineer.config.LabelingConfig); " - "ml4t.engineer.labeling.horizon_labels for fixed_time_horizon_labels and " - "trend_scanning_labels; " - "ml4t.engineer.labeling.uniqueness for build_concurrency, " - "calculate_label_uniqueness, calculate_sample_weights, and sequential_bootstrap. " - "Legacy helpers apply_triple_barrier, calculate_returns, and " - "compute_barrier_touches are no longer supported." -) - -raise ImportError(_REMOVAL_MESSAGE) diff --git a/src/ml4t/engineer/pipeline/__init__.py b/src/ml4t/engineer/pipeline/__init__.py deleted file mode 100644 index d3359b5..0000000 --- a/src/ml4t/engineer/pipeline/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Pipeline module for ml4t.engineer. - -Provides the DAG-based pipeline engine for feature engineering. -""" - -from ml4t.engineer.pipeline.engine import Pipeline, PipelineStep - -__all__ = ["Pipeline", "PipelineStep"] diff --git a/src/ml4t/engineer/pipeline/engine.py b/src/ml4t/engineer/pipeline/engine.py deleted file mode 100644 index 58c9d34..0000000 --- a/src/ml4t/engineer/pipeline/engine.py +++ /dev/null @@ -1,259 +0,0 @@ -"""Pipeline engine for ml4t.engineer. - -Provides the DAG-based execution engine for feature engineering pipelines. -""" - -from collections.abc import Callable -from dataclasses import dataclass, field -from typing import Any - -import polars as pl - - -@dataclass -class PipelineStep: - """Represents a single step in a pipeline. - - Parameters - ---------- - name : str - The name of this step - func : Callable - The function to execute - params : dict - Parameters to pass to the function - dependencies : list[str] - Names of steps this step depends on - """ - - name: str - func: Callable[..., Any] - params: dict[str, Any] = field(default_factory=dict) - dependencies: list[str] = field(default_factory=list) - - -class Pipeline: - """DAG-based pipeline for feature engineering. - - The pipeline executes a series of transformations on data in dependency order. - Each step can depend on the output of previous steps. - - Parameters - ---------- - steps : list[tuple[str, Callable] | PipelineStep] - List of pipeline steps. Each step can be either: - - A tuple of (name, function) - - A PipelineStep instance - - Examples - -------- - >>> from ml4t.engineer import pipeline - >>> import polars as pl - >>> - >>> # Create a simple pipeline - >>> pipe = pipeline.Pipeline(steps=[ - ... ("returns", lambda df: df.with_columns( - ... returns=pl.col("close").pct_change() - ... )), - ... ("volatility", lambda df: df.with_columns( - ... volatility=pl.col("returns").rolling_std(20) - ... )) - ... ]) - >>> - >>> # Run the pipeline - >>> result = pipeline.run(data) - """ - - def __init__( - self, - steps: list[tuple[str, Callable[..., Any]] | PipelineStep], - ): - """Initialize the pipeline.""" - self.steps: list[PipelineStep] = [] - self._results: dict[str, pl.DataFrame] = {} - - # Convert tuples to PipelineStep objects - for step in steps: - if isinstance(step, tuple): - name: str = step[0] - func: Callable[..., Any] = step[1] - self.steps.append(PipelineStep(name=name, func=func)) - elif isinstance(step, PipelineStep): - self.steps.append(step) - else: - raise ValueError( - f"Step must be tuple or PipelineStep, got {type(step)}", - ) - - # Validate DAG (check for cycles) - self._validate_dag() - - def _validate_dag(self) -> None: - """Validate that the pipeline forms a valid DAG (no cycles).""" - # Check that dependencies exist - step_names = {step.name for step in self.steps} - for step in self.steps: - for dep in step.dependencies: - if dep not in step_names: - raise ValueError( - f"Step '{step.name}' depends on unknown step '{dep}'", - ) - - # Check for cycles using DFS - self._detect_cycles() - - def _detect_cycles(self) -> None: - """Detect cycles in the dependency graph using DFS. - - Raises - ------ - ValueError - If a cycle is detected in the dependency graph - """ - # Build adjacency list for dependency graph - graph: dict[str, list[str]] = {step.name: step.dependencies for step in self.steps} - - # Track visit states: 0=unvisited, 1=visiting, 2=visited - visit_state: dict[str, int] = {step.name: 0 for step in self.steps} - - def dfs_visit(node: str, path: list[str]) -> None: - if visit_state[node] == 1: # Currently visiting - cycle detected - cycle_start = path.index(node) - cycle = path[cycle_start:] + [node] - raise ValueError(f"Cycle detected in pipeline: {' -> '.join(cycle)}") - - if visit_state[node] == 2: # Already visited - return - - visit_state[node] = 1 # Mark as visiting - path.append(node) - - for dependency in graph[node]: - dfs_visit(dependency, path) - - path.pop() - visit_state[node] = 2 # Mark as visited - - # Check each unvisited node - for step_name in graph: - if visit_state[step_name] == 0: - dfs_visit(step_name, []) - - def _get_execution_order(self) -> list[PipelineStep]: - """Get the order in which steps should be executed. - - Uses Kahn's algorithm for topological sorting to determine execution order. - Steps with no dependencies are executed first, followed by steps whose - dependencies have all been satisfied. - - IMPORTANT: Preserves original step order for steps with equal priority - (i.e., when no dependency constraints exist). - - Returns - ------- - list[PipelineStep] - Steps ordered by their dependencies (topologically sorted) - """ - # Create step lookup and index mapping for original order - step_lookup = {step.name: step for step in self.steps} - step_index = {step.name: i for i, step in enumerate(self.steps)} - - # Build in-degree count for each step - in_degree = {step.name: len(step.dependencies) for step in self.steps} - - # Initialize queue with steps that have no dependencies (preserve original order) - queue = [step_name for step_name, count in in_degree.items() if count == 0] - result = [] - - # Process steps in topological order - while queue: - # Sort by original order to preserve insertion sequence - queue.sort(key=lambda name: step_index[name]) - current_step_name = queue.pop(0) - result.append(step_lookup[current_step_name]) - - # For each step that depends on the current step, reduce its in-degree - for step in self.steps: - if current_step_name in step.dependencies: - in_degree[step.name] -= 1 - if in_degree[step.name] == 0: - queue.append(step.name) - - # Verify all steps were processed (should never happen after cycle detection) - if len(result) != len(self.steps): - remaining = [name for name, count in in_degree.items() if count > 0] - raise ValueError(f"Unable to resolve dependencies for steps: {remaining}") - - return result - - def run(self, data: pl.DataFrame) -> pl.DataFrame: - """Execute the pipeline on the input data. - - Parameters - ---------- - data : pl.DataFrame - The input data - - Returns - ------- - pl.DataFrame - The transformed data after all pipeline steps - """ - result = data - self._results = {"input": data} - - # Execute steps in order - for step in self._get_execution_order(): - # Apply the transformation - result = step.func(result, **step.params) if step.params else step.func(result) - - # Store intermediate result - self._results[step.name] = result - - return result - - def get_intermediate_result(self, step_name: str) -> pl.DataFrame | None: - """Get the result after a specific step. - - Parameters - ---------- - step_name : str - The name of the step - - Returns - ------- - pl.DataFrame or None - The data after the specified step, or None if not found - """ - return self._results.get(step_name) - - def add_step( - self, - step: tuple[str, Callable[..., Any]] | PipelineStep, - ) -> "Pipeline": - """Add a step to the pipeline. - - Parameters - ---------- - step : tuple or PipelineStep - The step to add - - Returns - ------- - Pipeline - Self for method chaining - """ - if isinstance(step, tuple): - step_name: str = step[0] - step_func: Callable[..., Any] = step[1] - self.steps.append(PipelineStep(name=step_name, func=step_func)) - elif isinstance(step, PipelineStep): - self.steps.append(step) - else: - raise ValueError(f"Step must be tuple or PipelineStep, got {type(step)}") - - self._validate_dag() - return self - - -__all__ = ["Pipeline", "PipelineStep"] diff --git a/src/ml4t/engineer/preprocessing.py b/src/ml4t/engineer/preprocessing.py index 95620f7..16a1e3c 100644 --- a/src/ml4t/engineer/preprocessing.py +++ b/src/ml4t/engineer/preprocessing.py @@ -519,7 +519,7 @@ class PreprocessingPipeline: Parameters ---------- recommendations : dict | None - Feature recommendations from EngineerConfig.to_dict(). + Feature recommendations from FeatureEvaluatorConfig (ml4t-diagnostic). Format: {"feature_name": {"transform": "standardize", "confidence": 0.9}} min_confidence : float, default 0.0 Minimum confidence threshold for applying recommendations. @@ -571,7 +571,7 @@ def from_recommendations( Parameters ---------- recommendations : dict - Output from EngineerConfig.to_dict() or similar format. + Output from FeatureEvaluatorConfig (ml4t-diagnostic) or similar format. Expected structure: {"feature": {"transform": "...", "confidence": ...}} min_confidence : float, default 0.0 Minimum confidence threshold. diff --git a/src/ml4t/engineer/selection/__init__.py b/src/ml4t/engineer/selection/__init__.py deleted file mode 100644 index 401766b..0000000 --- a/src/ml4t/engineer/selection/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Feature selection has moved to ml4t-diagnostic. - -Use ``ml4t.diagnostic.selection`` instead:: - - from ml4t.diagnostic.selection import FeatureSelector - -Install with: ``pip install ml4t-diagnostic`` -""" - -_MOVED_EXPORTS = { - "FeatureSelector", - "SelectionReport", - "SelectionStep", -} - - -def __getattr__(name: str) -> object: - if name in _MOVED_EXPORTS: - raise ImportError( - f"ml4t.engineer.selection.{name} has moved to ml4t-diagnostic. " - f"Use: from ml4t.diagnostic.selection import {name}" - ) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ml4t/engineer/validation/README.md b/src/ml4t/engineer/validation/README.md deleted file mode 100644 index b5bc0d8..0000000 --- a/src/ml4t/engineer/validation/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Cross-Validation for Financial Time Series - -## Important Notice - -Cross-validation with purging and embargo for financial time series is implemented in the **ml4t-backtest** library, not ml4t-engineer. - -## Why Not in ml4t-engineer? - -The ml4t-engineer library focuses on feature engineering, while ml4t-backtest specializes in backtesting and model evaluation. Proper cross-validation for financial time series requires: - -1. **Purging**: Removing training samples that are too close to test samples to prevent information leakage -2. **Embargo**: Adding a gap after test samples to account for the forward-looking nature of labels -3. **Label Horizons**: Accounting for how far into the future labels look - -These requirements are tightly coupled with backtesting and evaluation logic, making ml4t-backtest the natural home for these utilities. - -## Using Cross-Validation with ml4t-engineer Data - -To use proper cross-validation with data processed by ml4t-engineer: - -```python -# 1. Engineer features with ml4t-engineer -from ml4t.engineer import compute_features -from ml4t.engineer.labeling import triple_barrier_labels - -# Create features -result = compute_features(df, ["rsi", "adx"]) - -# Apply labeling -labeled_df = triple_barrier_labels( - df, - upper_barrier=0.02, - lower_barrier=0.01, - max_holding=10, -) - -# 2. Use ml4t-backtest for cross-validation (when available) -# See ml4t-backtest documentation for PurgedWalkForwardCV -``` - -## Available Cross-Validators in ml4t-backtest - -1. **PurgedWalkForwardCV**: Walk-forward cross-validation with purging and embargo - - Best for time series with strong temporal dependencies - - Supports expanding and rolling windows - -2. **CombinatorialPurgedKFold**: Combinatorial purged K-fold cross-validation - - Generates more training/test combinations - - Better for limited data scenarios - -## References - -- López de Prado, M. (2018). "Advances in Financial Machine Learning", Chapter 7: Cross-Validation in Finance -- Bailey, D. H., & López de Prado, M. (2012). "The Deflated Sharpe Ratio: Correcting for Selection Bias, Backtest Overfitting, and Non-Normality" - -## See Also - -- [ml4t-backtest documentation](https://pypi.org/project/ml4t-backtest/) for detailed usage examples -- [ml4t-engineer labeling module](../labeling/) for creating labels with proper horizons diff --git a/src/ml4t/engineer/validation/__init__.py b/src/ml4t/engineer/validation/__init__.py deleted file mode 100644 index 5ac793c..0000000 --- a/src/ml4t/engineer/validation/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Validation module for ml4t.engineer. - -IMPORTANT: Cross-validation with purging and embargo for financial time series -is implemented in the ml4t.eval library, not ml4t.engineer. - -Please use: - from ml4t.eval.splitters import PurgedWalkForwardCV, CombinatorialPurgedKFold - -See the ml4t.eval documentation for proper cross-validation in financial ML. -""" - -# No exports - see ml4t.eval for cross-validation utilities -__all__: list[str] = [] diff --git a/src/ml4t/engineer/validation/cv.py b/src/ml4t/engineer/validation/cv.py deleted file mode 100644 index 61a6cfe..0000000 --- a/src/ml4t/engineer/validation/cv.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Cross-validation utilities notice. - -IMPORTANT: Cross-validation with purging and embargo for financial time series -is implemented in the ml4t.eval library, not ml4t.engineer. - -The ml4t.eval library provides proper implementations of: -- PurgedWalkForwardCV: Walk-forward cross-validation with purging and embargo -- CombinatorialPurgedKFold: Combinatorial purged K-fold cross-validation - -These implementations correctly handle: -- Purging: Removing training samples that are too close to test samples -- Embargo: Adding a gap after test samples to prevent information leakage -- Label horizons: Accounting for the forward-looking nature of labels - -For cross-validation in financial machine learning, please use: - from ml4t.evaluation.splitters import PurgedWalkForwardCV, CombinatorialPurgedKFold - -See the qeval documentation for usage examples. -""" - -__all__: list[str] = [] diff --git a/src/ml4t/engineer/visualization/__init__.py b/src/ml4t/engineer/visualization/__init__.py deleted file mode 100644 index 84886ee..0000000 --- a/src/ml4t/engineer/visualization/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Plot export utilities. - -For feature analysis visualizations (IC, importance, drift), -use ``ml4t.diagnostic.visualization``. - -Example -------- ->>> from ml4t.engineer.visualization import export_plot ->>> export_plot(fig, "analysis.png", dpi=300) -""" - -from ml4t.engineer.visualization.summary import export_plot - -__all__ = [ - "export_plot", -] diff --git a/src/ml4t/engineer/visualization/summary.py b/src/ml4t/engineer/visualization/summary.py deleted file mode 100644 index b3f0ec7..0000000 --- a/src/ml4t/engineer/visualization/summary.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Plot export utilities. - -For feature analysis visualizations (IC, importance, drift), -use ``ml4t.diagnostic.visualization``. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from matplotlib.figure import Figure - - -def export_plot( - fig: Figure, - output_path: str | Path, - dpi: int = 300, - bbox_inches: str = "tight", - **kwargs: Any, -) -> None: - """Export matplotlib figure to file. - - Saves figure to PNG or PDF with configurable quality settings. - - Parameters - ---------- - fig : Figure - Matplotlib figure to save. - output_path : str | Path - Output file path. Format determined by extension (.png, .pdf, etc.). - dpi : int, default 300 - Resolution in dots per inch. Higher values = better quality but larger files. - - 150: Draft quality - - 300: Publication quality (default) - - 600: High-resolution print - bbox_inches : str, default "tight" - Bounding box specification. "tight" removes extra whitespace. - **kwargs - Additional keyword arguments passed to fig.savefig(). - - Raises - ------ - ValueError - If output_path has unsupported extension. - OSError - If file cannot be written (permissions, disk space, etc.). - - Examples - -------- - >>> from ml4t.engineer.visualization import export_plot - >>> - >>> # Create and export plot - >>> fig = plt.figure() - >>> # ... create plot ... - >>> export_plot(fig, "analysis.png", dpi=300) - >>> - >>> # High-quality PDF - >>> export_plot(fig, "analysis.pdf", dpi=600) - """ - output_path = Path(output_path) - - # Validate extension - valid_extensions = {".png", ".pdf", ".svg", ".jpg", ".jpeg", ".eps", ".ps"} - if output_path.suffix.lower() not in valid_extensions: - msg = f"Unsupported format: {output_path.suffix}. Use one of {valid_extensions}" - raise ValueError(msg) - - # Create parent directory if needed - output_path.parent.mkdir(parents=True, exist_ok=True) - - # Save figure - try: - fig.savefig(output_path, dpi=dpi, bbox_inches=bbox_inches, **kwargs) - except OSError as e: - msg = f"Failed to save figure to {output_path}: {e}" - raise OSError(msg) from e diff --git a/tests/bars/test_imbalance_bars.py b/tests/bars/test_imbalance_bars.py index 601c40d..e1ae1e8 100644 --- a/tests/bars/test_imbalance_bars.py +++ b/tests/bars/test_imbalance_bars.py @@ -238,12 +238,6 @@ def test_init_valid_params(self): assert sampler.alpha == 0.2 assert sampler.initial_p_buy == 0.6 - def test_init_deprecated_initial_expectation(self): - """Test deprecation warning for initial_expectation.""" - with pytest.warns(DeprecationWarning, match="initial_expectation is deprecated"): - sampler = ImbalanceBarSampler(expected_ticks_per_bar=100, initial_expectation=5000.0) - assert sampler.expected_ticks_per_bar == 100 - def test_init_invalid_expected_ticks(self): """Test initialization fails with invalid expected_ticks_per_bar.""" with pytest.raises(ValueError, match="expected_ticks_per_bar must be positive"): diff --git a/tests/bars/test_run_bars.py b/tests/bars/test_run_bars.py index 4becc2d..9c49997 100644 --- a/tests/bars/test_run_bars.py +++ b/tests/bars/test_run_bars.py @@ -205,12 +205,6 @@ def test_init_valid(self): assert sampler.alpha == 0.1 assert sampler.initial_p_buy == 0.5 - def test_init_deprecated_initial_run_expectation(self): - """Test deprecation warning for initial_run_expectation.""" - with pytest.warns(DeprecationWarning, match="initial_run_expectation is deprecated"): - sampler = TickRunBarSampler(expected_ticks_per_bar=100, initial_run_expectation=15) - assert sampler.expected_ticks_per_bar == 100 - def test_init_invalid_ticks_zero(self): """Test initialization fails with zero expected ticks.""" with pytest.raises(ValueError, match="expected_ticks_per_bar must be positive"): @@ -342,14 +336,6 @@ def test_init_valid(self): assert sampler.expected_ticks_per_bar == 100 assert sampler.alpha == 0.1 - def test_init_deprecated_initial_run_expectation(self): - """Test deprecation warning for initial_run_expectation.""" - with pytest.warns(DeprecationWarning, match="initial_run_expectation is deprecated"): - sampler = VolumeRunBarSampler( - expected_ticks_per_bar=100, initial_run_expectation=1000.0 - ) - assert sampler.expected_ticks_per_bar == 100 - def test_init_invalid_ticks_zero(self): """Test initialization fails with zero expected ticks.""" with pytest.raises(ValueError, match="expected_ticks_per_bar must be positive"): @@ -443,14 +429,6 @@ def test_init_valid(self): assert sampler.expected_ticks_per_bar == 100 assert sampler.alpha == 0.1 - def test_init_deprecated_initial_run_expectation(self): - """Test deprecation warning for initial_run_expectation.""" - with pytest.warns(DeprecationWarning, match="initial_run_expectation is deprecated"): - sampler = DollarRunBarSampler( - expected_ticks_per_bar=100, initial_run_expectation=100000.0 - ) - assert sampler.expected_ticks_per_bar == 100 - def test_init_invalid_ticks_zero(self): """Test initialization fails with zero expected ticks.""" with pytest.raises(ValueError, match="expected_ticks_per_bar must be positive"): diff --git a/tests/bars/test_vectorized_bars.py b/tests/bars/test_vectorized_bars.py index acdc8dd..402f736 100644 --- a/tests/bars/test_vectorized_bars.py +++ b/tests/bars/test_vectorized_bars.py @@ -473,8 +473,8 @@ def test_sample_include_incomplete(self, sample_tick_data): assert len(bars_with) >= len(bars_without) - def test_initial_expectation_estimation(self, sample_tick_data): - """Test AFML parameters are estimated when not provided.""" + def test_afml_parameter_estimation(self, sample_tick_data): + """Test AFML parameters are estimated from data.""" sampler = ImbalanceBarSamplerVectorized(expected_ticks_per_bar=50) sampler.sample(sample_tick_data) @@ -484,16 +484,6 @@ def test_initial_expectation_estimation(self, sample_tick_data): assert sampler._initial_v_buy is not None assert sampler._initial_v_buy > 0 - def test_initial_expectation_provided(self, sample_tick_data): - """Test sampling with provided initial expectation.""" - sampler = ImbalanceBarSamplerVectorized( - expected_ticks_per_bar=50, initial_expectation=500.0 - ) - - sampler.sample(sample_tick_data) - # Should use provided initial_expectation - assert sampler.initial_expectation == 500.0 - def test_empty_result_schema(self): """Test _empty_imbalance_bars_df returns correct schema.""" sampler = ImbalanceBarSamplerVectorized(expected_ticks_per_bar=50) diff --git a/tests/core/test_calendars.py b/tests/core/test_calendars.py index 858d086..e4282c8 100644 --- a/tests/core/test_calendars.py +++ b/tests/core/test_calendars.py @@ -148,16 +148,16 @@ def test_next_open_from_market_hours(self) -> None: """Test next open when currently in market hours.""" calendar = EquityCalendar() - # Tuesday 10:00 AM ET - dt = datetime(2024, 1, 2, 10, 0, 0) + # Wednesday 10:00 AM ET (during market hours) + dt = datetime(2024, 1, 3, 10, 0, 0) dt = dt.replace(tzinfo=ZoneInfo("America/New_York")) next_open = calendar.next_open(dt) # Convert to ET for comparison next_open_et = next_open.astimezone(ZoneInfo("America/New_York")) - # Next open should be Wednesday 9:30 AM ET - assert next_open_et.day == 3 + # Next open should be Thursday 9:30 AM ET + assert next_open_et.day == 4 assert next_open_et.hour == 9 assert next_open_et.minute == 30 @@ -199,17 +199,16 @@ def test_previous_close_from_market_hours(self) -> None: """Test previous close when in market hours.""" calendar = EquityCalendar() - # Tuesday 10:00 AM ET - dt = datetime(2024, 1, 2, 10, 0, 0) + # Wednesday 10:00 AM ET (avoids holiday ambiguity) + dt = datetime(2024, 1, 3, 10, 0, 0) dt = dt.replace(tzinfo=ZoneInfo("America/New_York")) prev_close = calendar.previous_close(dt) # Convert to ET for comparison prev_close_et = prev_close.astimezone(ZoneInfo("America/New_York")) - # Previous close should be Dec 29, 2023 4:00 PM (Jan 1 is a holiday) - assert prev_close_et.month == 12 - assert prev_close_et.day == 29 + # Previous close should be Tuesday Jan 2, 4:00 PM + assert prev_close_et.day == 2 assert prev_close_et.hour == 16 assert prev_close_et.minute == 0 @@ -217,24 +216,24 @@ def test_previous_close_from_before_open(self) -> None: """Test previous close when before market open.""" calendar = EquityCalendar() - # Tuesday 8:00 AM ET (before open) - dt = datetime(2024, 1, 2, 8, 0, 0) + # Wednesday 8:00 AM ET (before open, avoids holiday ambiguity) + dt = datetime(2024, 1, 3, 8, 0, 0) dt = dt.replace(tzinfo=ZoneInfo("America/New_York")) prev_close = calendar.previous_close(dt) # Convert to ET for comparison prev_close_et = prev_close.astimezone(ZoneInfo("America/New_York")) - # Previous close should be Dec 29, 2023 4:00 PM (Jan 1 is a holiday) - assert prev_close_et.month == 12 - assert prev_close_et.day == 29 + # Previous close should be Tuesday Jan 2, 4:00 PM + assert prev_close_et.day == 2 assert prev_close_et.hour == 16 def test_previous_close_skips_weekend(self) -> None: """Test previous close skips weekend.""" calendar = EquityCalendar() - # Monday 10:00 AM ET + # Monday 10:00 AM ET — during market hours, so previous close is + # the last trading day's close, which should skip the weekend monday = datetime(2024, 1, 8, 10, 0, 0) monday = monday.replace(tzinfo=ZoneInfo("America/New_York")) @@ -242,8 +241,9 @@ def test_previous_close_skips_weekend(self) -> None: # Convert to ET for comparison prev_close_et = prev_close.astimezone(ZoneInfo("America/New_York")) - # Should be previous Friday 4:00 PM ET + # Should be previous Friday (Jan 5) 4:00 PM ET assert prev_close_et.weekday() == 4 # Friday + assert prev_close_et.day == 5 assert prev_close_et.hour == 16 @@ -259,18 +259,17 @@ def test_sessions_between_same_week(self) -> None: """Test sessions between dates in same week.""" calendar = EquityCalendar() - # Jan 1, 2024 is New Year's Day (holiday), so use Jan 2-5 - # which gives Tue-Fri = 4 sessions - start = datetime(2024, 1, 1, 9, 30, 0) # Monday (holiday) - end = datetime(2024, 1, 5, 16, 0, 0) # Friday + # Use Jan 8-12, 2024 (no holidays, Mon-Fri) + start = datetime(2024, 1, 8, 9, 30, 0) # Monday + end = datetime(2024, 1, 12, 16, 0, 0) # Friday start = start.replace(tzinfo=ZoneInfo("America/New_York")) end = end.replace(tzinfo=ZoneInfo("America/New_York")) sessions = calendar.sessions_between(start, end) - # Should have 4 sessions (Tue-Fri, skipping New Year's Day) - assert len(sessions) == 4 + # Should have 5 sessions (Mon-Fri) + assert len(sessions) == 5 # All should be weekdays for session in sessions: @@ -521,9 +520,12 @@ def test_timezone_aware_workflow(self) -> None: # Should detect as in session assert calendar.is_session(dt_utc) is True - # Next open in UTC should be next day at correct time + # Next open should be next day (currently in session) next_open = calendar.next_open(dt_utc) - assert next_open.day == 3 # Next day + next_open_et = next_open.astimezone(ZoneInfo("America/New_York")) + assert next_open_et.day == 3 # Next day + assert next_open_et.hour == 9 + assert next_open_et.minute == 30 # ============================================================================= diff --git a/tests/features/volatility/test_volatility_comprehensive.py b/tests/features/volatility/test_volatility_comprehensive.py new file mode 100644 index 0000000..ef1158e --- /dev/null +++ b/tests/features/volatility/test_volatility_comprehensive.py @@ -0,0 +1,950 @@ +"""Comprehensive tests for non-TA-Lib volatility indicators. + +Tests mathematical properties, edge cases, parameter variations, +and relationships between estimators for all 11 Polars-based +volatility features. +""" + +import numpy as np +import polars as pl +import pytest + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def ohlcv_df(): + """Deterministic OHLCV DataFrame (100 rows) with known properties.""" + np.random.seed(42) + n = 100 + # Geometric brownian motion for realistic prices + returns = np.random.randn(n) * 0.02 # ~2% daily vol + close = 100.0 * np.exp(np.cumsum(returns)) + high = close * (1 + np.abs(np.random.randn(n) * 0.005)) + low = close * (1 - np.abs(np.random.randn(n) * 0.005)) + open_ = np.roll(close, 1) * (1 + np.random.randn(n) * 0.002) + open_[0] = close[0] + # Ensure OHLC consistency + high = np.maximum(high, np.maximum(open_, close)) + low = np.minimum(low, np.minimum(open_, close)) + return pl.DataFrame( + { + "open": open_, + "high": high, + "low": low, + "close": close, + "volume": np.random.randint(100, 10000, n).astype(float), + } + ) + + +@pytest.fixture +def constant_df(): + """OHLCV DataFrame with constant prices (zero volatility).""" + n = 60 + return pl.DataFrame( + { + "open": [100.0] * n, + "high": [100.0] * n, + "low": [100.0] * n, + "close": [100.0] * n, + "volume": [1000.0] * n, + } + ) + + +@pytest.fixture +def trending_df(): + """OHLCV with strong upward drift (tests drift-independence).""" + n = 100 + close = np.linspace(100, 200, n) # 100% drift over 100 bars + high = close * 1.005 + low = close * 0.995 + open_ = np.roll(close, 1) + open_[0] = close[0] + high = np.maximum(high, np.maximum(open_, close)) + low = np.minimum(low, np.minimum(open_, close)) + return pl.DataFrame( + { + "open": open_, + "high": high, + "low": low, + "close": close, + "volume": [1000.0] * n, + } + ) + + +# --------------------------------------------------------------------------- +# 1. Parkinson Volatility +# --------------------------------------------------------------------------- + + +class TestParkinsonVolatility: + """Tests for parkinson_volatility.""" + + def test_basic_output(self, ohlcv_df): + """Runs on deterministic data, validates output shape/type.""" + from ml4t.engineer.features.volatility.parkinson_volatility import parkinson_volatility + + result = ohlcv_df.select(parkinson_volatility("high", "low", period=20).alias("pv")) + assert result.shape == (100, 1) + assert result.dtypes[0] == pl.Float64 + + def test_positivity(self, ohlcv_df): + """Parkinson volatility must be non-negative.""" + from ml4t.engineer.features.volatility.parkinson_volatility import parkinson_volatility + + vals = ( + ohlcv_df.select(parkinson_volatility("high", "low", period=20).alias("pv"))["pv"] + .drop_nulls() + .drop_nans() + ) + assert (vals >= 0).all() + + def test_zero_range_gives_zero(self, constant_df): + """H=L => ln(H/L)=0 => zero volatility.""" + from ml4t.engineer.features.volatility.parkinson_volatility import parkinson_volatility + + vals = ( + constant_df.select(parkinson_volatility("high", "low", period=10).alias("pv"))["pv"] + .drop_nulls() + .drop_nans() + ) + assert len(vals) > 0 + assert (vals.abs() < 1e-10).all() + + def test_different_periods(self, ohlcv_df): + """Longer period smooths more.""" + from ml4t.engineer.features.volatility.parkinson_volatility import parkinson_volatility + + short = ( + ohlcv_df.select(parkinson_volatility("high", "low", period=10).alias("pv"))["pv"] + .drop_nulls() + .drop_nans() + ) + long = ( + ohlcv_df.select(parkinson_volatility("high", "low", period=30).alias("pv"))["pv"] + .drop_nulls() + .drop_nans() + ) + # Longer period should have lower variance (smoothing effect) + assert short.std() >= long.std() * 0.5 # Generous bound + + def test_annualization(self, ohlcv_df): + """Annualized > non-annualized by sqrt(252) factor.""" + from ml4t.engineer.features.volatility.parkinson_volatility import parkinson_volatility + + ann = ( + ohlcv_df.select( + parkinson_volatility("high", "low", period=20, annualize=True).alias("v") + )["v"] + .drop_nulls() + .drop_nans() + ) + raw = ( + ohlcv_df.select( + parkinson_volatility("high", "low", period=20, annualize=False).alias("v") + )["v"] + .drop_nulls() + .drop_nans() + ) + # Annualized should be ~sqrt(252) times larger + ratio = ann.mean() / raw.mean() + assert 10 < ratio < 20 # sqrt(252) ~ 15.87 + + +# --------------------------------------------------------------------------- +# 2. Garman-Klass Volatility +# --------------------------------------------------------------------------- + + +class TestGarmanKlassVolatility: + """Tests for garman_klass_volatility.""" + + def test_basic_output(self, ohlcv_df): + """Runs on deterministic data, validates output shape.""" + from ml4t.engineer.features.volatility.garman_klass_volatility import ( + garman_klass_volatility, + ) + + result = ohlcv_df.select( + garman_klass_volatility("open", "high", "low", "close", period=20).alias("gk") + ) + assert result.shape == (100, 1) + + def test_positivity(self, ohlcv_df): + """Garman-Klass volatility must be non-negative.""" + from ml4t.engineer.features.volatility.garman_klass_volatility import ( + garman_klass_volatility, + ) + + vals = ( + ohlcv_df.select( + garman_klass_volatility("open", "high", "low", "close", period=20).alias("gk") + )["gk"] + .drop_nulls() + .drop_nans() + ) + assert (vals >= 0).all() + + def test_more_efficient_than_parkinson(self, ohlcv_df): + """GK uses more information (OHLC vs HL) and should be more stable.""" + from ml4t.engineer.features.volatility.garman_klass_volatility import ( + garman_klass_volatility, + ) + from ml4t.engineer.features.volatility.parkinson_volatility import parkinson_volatility + + gk = ( + ohlcv_df.select( + garman_klass_volatility("open", "high", "low", "close", period=20).alias("v") + )["v"] + .drop_nulls() + .drop_nans() + ) + pk = ( + ohlcv_df.select(parkinson_volatility("high", "low", period=20).alias("v"))["v"] + .drop_nulls() + .drop_nans() + ) + # Both should produce similar magnitudes (same underlying data) + assert abs(gk.mean() - pk.mean()) / pk.mean() < 1.0 # Within 100% + + def test_constant_prices(self, constant_df): + """Constant prices => zero volatility.""" + from ml4t.engineer.features.volatility.garman_klass_volatility import ( + garman_klass_volatility, + ) + + vals = ( + constant_df.select( + garman_klass_volatility("open", "high", "low", "close", period=10).alias("gk") + )["gk"] + .drop_nulls() + .drop_nans() + ) + assert len(vals) > 0 + assert (vals.abs() < 1e-10).all() + + +# --------------------------------------------------------------------------- +# 3. Rogers-Satchell Volatility +# --------------------------------------------------------------------------- + + +class TestRogersSatchellVolatility: + """Tests for rogers_satchell_volatility.""" + + def test_basic_output(self, ohlcv_df): + """Validates output shape and type.""" + from ml4t.engineer.features.volatility.rogers_satchell_volatility import ( + rogers_satchell_volatility, + ) + + result = ohlcv_df.select( + rogers_satchell_volatility("open", "high", "low", "close", period=20).alias("rs") + ) + assert result.shape == (100, 1) + + def test_drift_independence(self, trending_df, ohlcv_df): + """RS is drift-independent; trending data shouldn't inflate vol estimate.""" + from ml4t.engineer.features.volatility.rogers_satchell_volatility import ( + rogers_satchell_volatility, + ) + + # Trending data with similar intraday range should give similar vol + rs_trend = ( + trending_df.select( + rogers_satchell_volatility( + "open", "high", "low", "close", period=20, annualize=False + ).alias("v") + )["v"] + .drop_nulls() + .drop_nans() + ) + # Main check: RS on trending data shouldn't blow up + assert rs_trend.mean() < 1.0 # Sanity — shouldn't be massive + + def test_positivity(self, ohlcv_df): + """RS should produce non-negative values for well-formed OHLC.""" + from ml4t.engineer.features.volatility.rogers_satchell_volatility import ( + rogers_satchell_volatility, + ) + + vals = ( + ohlcv_df.select( + rogers_satchell_volatility("open", "high", "low", "close", period=20).alias("rs") + )["rs"] + .drop_nulls() + .drop_nans() + ) + assert (vals >= 0).all() + + def test_different_periods(self, ohlcv_df): + """Test with different window sizes.""" + from ml4t.engineer.features.volatility.rogers_satchell_volatility import ( + rogers_satchell_volatility, + ) + + for period in [10, 20, 50]: + result = ohlcv_df.select( + rogers_satchell_volatility("open", "high", "low", "close", period=period).alias("v") + ) + assert len(result) == 100 + + +# --------------------------------------------------------------------------- +# 4. Yang-Zhang Volatility +# --------------------------------------------------------------------------- + + +class TestYangZhangVolatility: + """Tests for yang_zhang_volatility.""" + + def test_basic_output(self, ohlcv_df): + """Validates output shape and type.""" + from ml4t.engineer.features.volatility.yang_zhang_volatility import yang_zhang_volatility + + result = ohlcv_df.select( + yang_zhang_volatility("open", "high", "low", "close", period=20).alias("yz") + ) + assert result.shape == (100, 1) + + def test_positivity(self, ohlcv_df): + """Yang-Zhang should be non-negative.""" + from ml4t.engineer.features.volatility.yang_zhang_volatility import yang_zhang_volatility + + vals = ( + ohlcv_df.select( + yang_zhang_volatility("open", "high", "low", "close", period=20).alias("yz") + )["yz"] + .drop_nulls() + .drop_nans() + ) + assert (vals >= 0).all() + + def test_combines_overnight_and_intraday(self, ohlcv_df): + """YZ should capture both overnight and intraday moves.""" + from ml4t.engineer.features.volatility.yang_zhang_volatility import yang_zhang_volatility + + vals = ( + ohlcv_df.select( + yang_zhang_volatility("open", "high", "low", "close", period=20).alias("yz") + )["yz"] + .drop_nulls() + .drop_nans() + ) + # Should produce non-trivial estimates + assert vals.mean() > 0 + + def test_constant_prices(self, constant_df): + """Constant prices => zero volatility.""" + from ml4t.engineer.features.volatility.yang_zhang_volatility import yang_zhang_volatility + + vals = ( + constant_df.select( + yang_zhang_volatility("open", "high", "low", "close", period=20).alias("yz") + )["yz"] + .drop_nulls() + .drop_nans() + ) + assert len(vals) > 0 + assert (vals.abs() < 1e-10).all() + + +# --------------------------------------------------------------------------- +# 5. Realized Volatility +# --------------------------------------------------------------------------- + + +class TestRealizedVolatility: + """Tests for realized_volatility.""" + + def test_basic_output(self, ohlcv_df): + """Validates output shape — note: takes returns, not close.""" + from ml4t.engineer.features.volatility.realized_volatility import realized_volatility + + df = ohlcv_df.with_columns(returns=pl.col("close").pct_change()) + result = df.select(realized_volatility("returns", period=20).alias("rv")) + assert result.shape == (100, 1) + + def test_constant_returns_zero(self): + """Constant returns => zero realized vol.""" + from ml4t.engineer.features.volatility.realized_volatility import realized_volatility + + df = pl.DataFrame({"returns": [0.01] * 50}) + vals = ( + df.select(realized_volatility("returns", period=10, annualize=False).alias("rv"))["rv"] + .drop_nulls() + .drop_nans() + ) + assert len(vals) > 0 + assert (vals.abs() < 1e-10).all() + + def test_annualization_factor(self, ohlcv_df): + """Annualized should be sqrt(trading_periods) times raw.""" + from ml4t.engineer.features.volatility.realized_volatility import realized_volatility + + df = ohlcv_df.with_columns(returns=pl.col("close").pct_change()) + ann = ( + df.select( + realized_volatility( + "returns", period=20, annualize=True, trading_periods=252 + ).alias("v") + )["v"] + .drop_nulls() + .drop_nans() + ) + raw = ( + df.select(realized_volatility("returns", period=20, annualize=False).alias("v"))["v"] + .drop_nulls() + .drop_nans() + ) + ratio = ann.mean() / raw.mean() + expected = np.sqrt(252) + assert abs(ratio - expected) / expected < 0.01 # Within 1% + + def test_positivity(self, ohlcv_df): + """Realized vol is always non-negative.""" + from ml4t.engineer.features.volatility.realized_volatility import realized_volatility + + df = ohlcv_df.with_columns(returns=pl.col("close").pct_change()) + vals = ( + df.select(realized_volatility("returns", period=20).alias("rv"))["rv"] + .drop_nulls() + .drop_nans() + ) + assert (vals >= 0).all() + + +# --------------------------------------------------------------------------- +# 6. EWMA Volatility +# --------------------------------------------------------------------------- + + +class TestEWMAVolatility: + """Tests for ewma_volatility.""" + + def test_basic_output(self, ohlcv_df): + """Validates output shape.""" + from ml4t.engineer.features.volatility.ewma_volatility import ewma_volatility + + result = ohlcv_df.select(ewma_volatility("close", span=20).alias("ev")) + assert result.shape == (100, 1) + + def test_different_spans(self, ohlcv_df): + """Shorter span reacts faster to recent data.""" + from ml4t.engineer.features.volatility.ewma_volatility import ewma_volatility + + short = ( + ohlcv_df.select(ewma_volatility("close", span=10).alias("v"))["v"] + .drop_nulls() + .drop_nans() + ) + long = ( + ohlcv_df.select(ewma_volatility("close", span=50).alias("v"))["v"] + .drop_nulls() + .drop_nans() + ) + # Shorter span should have higher variance (more reactive) + assert short.std() >= long.std() * 0.3 # Generous bound + + def test_positivity(self, ohlcv_df): + """EWMA vol is always non-negative.""" + from ml4t.engineer.features.volatility.ewma_volatility import ewma_volatility + + vals = ( + ohlcv_df.select(ewma_volatility("close", span=20).alias("ev"))["ev"] + .drop_nulls() + .drop_nans() + ) + assert (vals >= 0).all() + + def test_normalized_output(self, ohlcv_df): + """Normalized output should be in [-1, 1] range.""" + from ml4t.engineer.features.volatility.ewma_volatility import ewma_volatility + + vals = ( + ohlcv_df.select(ewma_volatility("close", span=20, normalize=True).alias("ev"))["ev"] + .drop_nulls() + .drop_nans() + ) + assert (vals >= -1.0).all() + assert (vals <= 1.0).all() + + +# --------------------------------------------------------------------------- +# 7. GARCH Forecast +# --------------------------------------------------------------------------- + + +class TestGARCHForecast: + """Tests for garch_forecast.""" + + def test_basic_output(self, ohlcv_df): + """Validates output shape — takes returns.""" + from ml4t.engineer.features.volatility.garch_forecast import garch_forecast + + df = ohlcv_df.with_columns(returns=pl.col("close").pct_change()) + result = df.select(garch_forecast("returns").alias("gf")) + assert result.shape == (100, 1) + + def test_alpha_beta_constraint(self): + """alpha + beta must be < 1 for stationarity.""" + from ml4t.engineer.features.volatility.garch_forecast import garch_forecast + + with pytest.raises(ValueError): + pl.DataFrame({"r": [0.01] * 10}).select(garch_forecast("r", alpha=0.5, beta=0.6)) + + def test_responds_to_shocks(self, ohlcv_df): + """Conditional vol should increase after large returns.""" + from ml4t.engineer.features.volatility.garch_forecast import garch_forecast + + # Create data with a shock + returns = [0.001] * 30 + [0.10] + [0.001] * 30 # big shock at t=30 + df = pl.DataFrame({"returns": returns}) + vals = df.select(garch_forecast("returns").alias("gf"))["gf"].to_numpy() + # After shock, vol should be higher than before + pre_shock = vals[25:30] + post_shock = vals[31:36] + pre_mean = np.nanmean(pre_shock) + post_mean = np.nanmean(post_shock) + assert post_mean > pre_mean + + def test_positivity(self, ohlcv_df): + """GARCH forecasts should be non-negative.""" + from ml4t.engineer.features.volatility.garch_forecast import garch_forecast + + df = ohlcv_df.with_columns(returns=pl.col("close").pct_change()) + vals = df.select(garch_forecast("returns").alias("gf"))["gf"].drop_nulls().drop_nans() + assert (vals >= 0).all() + + def test_parameter_validation(self): + """Invalid parameters should raise.""" + from ml4t.engineer.features.volatility.garch_forecast import garch_forecast + + df = pl.DataFrame({"r": [0.01] * 10}) + with pytest.raises(ValueError): + df.select(garch_forecast("r", omega=-0.001)) + with pytest.raises(ValueError): + df.select(garch_forecast("r", alpha=-0.1)) + with pytest.raises(ValueError): + df.select(garch_forecast("r", beta=-0.1)) + with pytest.raises(ValueError): + df.select(garch_forecast("r", horizon=0)) + + +# --------------------------------------------------------------------------- +# 8. Conditional Volatility Ratio +# --------------------------------------------------------------------------- + + +class TestConditionalVolatilityRatio: + """Tests for conditional_volatility_ratio.""" + + def test_basic_output(self, ohlcv_df): + """Validates output shape.""" + from ml4t.engineer.features.volatility.conditional_volatility_ratio import ( + conditional_volatility_ratio, + ) + + df = ohlcv_df.with_columns(returns=pl.col("close").pct_change()) + result = df.select(conditional_volatility_ratio("returns", period=20).alias("cvr")) + assert result.shape == (100, 1) + + def test_symmetric_returns_near_one(self): + """Symmetric returns should give ratio close to 1.""" + from ml4t.engineer.features.volatility.conditional_volatility_ratio import ( + conditional_volatility_ratio, + ) + + np.random.seed(42) + # Symmetric normal returns + returns = np.random.randn(200) * 0.01 + df = pl.DataFrame({"returns": returns}) + vals = ( + df.select(conditional_volatility_ratio("returns", period=50).alias("cvr"))["cvr"] + .drop_nulls() + .drop_nans() + ) + # Mean should be close to 1.0 for symmetric distributions + assert 0.5 < vals.mean() < 2.0 + + def test_positivity(self, ohlcv_df): + """Ratio should be non-negative.""" + from ml4t.engineer.features.volatility.conditional_volatility_ratio import ( + conditional_volatility_ratio, + ) + + df = ohlcv_df.with_columns(returns=pl.col("close").pct_change()) + vals = ( + df.select(conditional_volatility_ratio("returns", period=20).alias("cvr"))["cvr"] + .drop_nulls() + .drop_nans() + ) + assert (vals >= 0).all() + + def test_different_thresholds(self, ohlcv_df): + """Different thresholds should produce different decompositions.""" + from ml4t.engineer.features.volatility.conditional_volatility_ratio import ( + conditional_volatility_ratio, + ) + + df = ohlcv_df.with_columns(returns=pl.col("close").pct_change()) + r0 = ( + df.select(conditional_volatility_ratio("returns", threshold=0.0, period=20).alias("v"))[ + "v" + ] + .drop_nulls() + .drop_nans() + ) + r1 = ( + df.select( + conditional_volatility_ratio("returns", threshold=0.01, period=20).alias("v") + )["v"] + .drop_nulls() + .drop_nans() + ) + # Higher threshold means fewer "upside" returns, ratio should differ + assert r0.mean() != r1.mean() + + +# --------------------------------------------------------------------------- +# 9. Volatility of Volatility +# --------------------------------------------------------------------------- + + +class TestVolatilityOfVolatility: + """Tests for volatility_of_volatility.""" + + def test_basic_output(self, ohlcv_df): + """Validates output shape.""" + from ml4t.engineer.features.volatility.volatility_of_volatility import ( + volatility_of_volatility, + ) + + result = ohlcv_df.select( + volatility_of_volatility("close", vol_period=10, vov_period=10).alias("vov") + ) + assert result.shape == (100, 1) + + def test_constant_vol_near_zero(self): + """Constant volatility input => near-zero VoV.""" + from ml4t.engineer.features.volatility.volatility_of_volatility import ( + volatility_of_volatility, + ) + + # Very slight random walk with nearly constant vol + np.random.seed(42) + close = 100 + np.cumsum(np.full(200, 0.01)) # constant small changes + df = pl.DataFrame({"close": close}) + vals = ( + df.select( + volatility_of_volatility( + "close", vol_period=10, vov_period=10, annualize=False + ).alias("vov") + )["vov"] + .drop_nulls() + .drop_nans() + ) + # Should be very small since underlying vol is nearly constant + assert vals.mean() < 0.01 + + def test_positivity(self, ohlcv_df): + """VoV should be non-negative.""" + from ml4t.engineer.features.volatility.volatility_of_volatility import ( + volatility_of_volatility, + ) + + vals = ( + ohlcv_df.select( + volatility_of_volatility("close", vol_period=10, vov_period=10).alias("vov") + )["vov"] + .drop_nulls() + .drop_nans() + ) + assert (vals >= 0).all() + + def test_different_periods(self, ohlcv_df): + """Different vol_period and vov_period combinations.""" + from ml4t.engineer.features.volatility.volatility_of_volatility import ( + volatility_of_volatility, + ) + + for vp, vvp in [(5, 5), (10, 10), (20, 20)]: + result = ohlcv_df.select( + volatility_of_volatility("close", vol_period=vp, vov_period=vvp).alias("vov") + ) + assert len(result) == 100 + + +# --------------------------------------------------------------------------- +# 10. Volatility Percentile Rank +# --------------------------------------------------------------------------- + + +class TestVolatilityPercentileRank: + """Tests for volatility_percentile_rank.""" + + def test_basic_output(self, ohlcv_df): + """Validates output shape.""" + from ml4t.engineer.features.volatility.volatility_percentile_rank import ( + volatility_percentile_rank, + ) + + result = ohlcv_df.select( + volatility_percentile_rank("close", period=10, lookback=50).alias("vpr") + ) + assert result.shape == (100, 1) + + def test_output_in_0_100(self, ohlcv_df): + """Output should be in [0, 100] range.""" + from ml4t.engineer.features.volatility.volatility_percentile_rank import ( + volatility_percentile_rank, + ) + + vals = ( + ohlcv_df.select( + volatility_percentile_rank("close", period=10, lookback=50).alias("vpr") + )["vpr"] + .drop_nulls() + .drop_nans() + ) + assert len(vals) > 0 + assert (vals >= 0.0).all() + assert (vals <= 100.0).all() + + def test_different_lookbacks(self, ohlcv_df): + """Shorter lookback = more responsive to recent vol changes.""" + from ml4t.engineer.features.volatility.volatility_percentile_rank import ( + volatility_percentile_rank, + ) + + short = ( + ohlcv_df.select(volatility_percentile_rank("close", period=10, lookback=30).alias("v"))[ + "v" + ] + .drop_nulls() + .drop_nans() + ) + long = ( + ohlcv_df.select(volatility_percentile_rank("close", period=10, lookback=80).alias("v"))[ + "v" + ] + .drop_nulls() + .drop_nans() + ) + # Both should be bounded + assert (short >= 0).all() and (short <= 100).all() + assert (long >= 0).all() and (long <= 100).all() + + +# --------------------------------------------------------------------------- +# 11. Volatility Regime Probability +# --------------------------------------------------------------------------- + + +class TestVolatilityRegimeProbability: + """Tests for volatility_regime_probability.""" + + def test_basic_output(self, ohlcv_df): + """Validates that function returns dict with correct keys.""" + from ml4t.engineer.features.volatility.volatility_regime_probability import ( + volatility_regime_probability, + ) + + exprs = volatility_regime_probability("close", period=10, lookback=50) + assert isinstance(exprs, dict) + assert "prob_low_vol" in exprs + assert "prob_med_vol" in exprs + assert "prob_high_vol" in exprs + assert "current_vol" in exprs + + def test_probabilities_sum_to_one(self, ohlcv_df): + """Low + med + high probabilities should sum to ~1.""" + from ml4t.engineer.features.volatility.volatility_regime_probability import ( + volatility_regime_probability, + ) + + exprs = volatility_regime_probability("close", period=10, lookback=50) + result = ohlcv_df.select( + exprs["prob_low_vol"].alias("low"), + exprs["prob_med_vol"].alias("med"), + exprs["prob_high_vol"].alias("high"), + ) + total = (result["low"] + result["med"] + result["high"]).drop_nulls().drop_nans() + assert len(total) > 0 + assert np.allclose(total.to_numpy(), 1.0, atol=0.01) + + def test_probabilities_in_0_1(self, ohlcv_df): + """Each probability should be in [0, 1].""" + from ml4t.engineer.features.volatility.volatility_regime_probability import ( + volatility_regime_probability, + ) + + exprs = volatility_regime_probability("close", period=10, lookback=50) + result = ohlcv_df.select( + exprs["prob_low_vol"].alias("low"), + exprs["prob_med_vol"].alias("med"), + exprs["prob_high_vol"].alias("high"), + ) + for col in ["low", "med", "high"]: + vals = result[col].drop_nulls().drop_nans() + assert (vals >= 0.0).all() + assert (vals <= 1.0).all() + + def test_threshold_ordering(self): + """low_vol_threshold must be < high_vol_threshold.""" + from ml4t.engineer.features.volatility.volatility_regime_probability import ( + volatility_regime_probability, + ) + + with pytest.raises(ValueError): + volatility_regime_probability("close", low_vol_threshold=0.05, high_vol_threshold=0.01) + + def test_current_vol_positive(self, ohlcv_df): + """Current vol should be non-negative.""" + from ml4t.engineer.features.volatility.volatility_regime_probability import ( + volatility_regime_probability, + ) + + exprs = volatility_regime_probability("close", period=10, lookback=50) + vals = ohlcv_df.select(exprs["current_vol"].alias("v"))["v"].drop_nulls().drop_nans() + assert (vals >= 0).all() + + +# --------------------------------------------------------------------------- +# Bollinger Bands (enhance existing coverage) +# --------------------------------------------------------------------------- + + +class TestBollingerBandsComprehensive: + """Enhanced tests for bollinger_bands.""" + + def test_band_ordering(self): + """Upper > middle > lower always.""" + from ml4t.engineer.features.volatility.bollinger_bands import bollinger_bands + + np.random.seed(42) + close = 100 + np.cumsum(np.random.randn(100) * 0.5) + upper, middle, lower = bollinger_bands(close, period=20, nbdevup=2.0, nbdevdn=2.0) + + # Compare only where all three are valid (non-NaN) + mask = ~(np.isnan(upper) | np.isnan(middle) | np.isnan(lower)) + assert np.all(upper[mask] >= middle[mask]) + assert np.all(middle[mask] >= lower[mask]) + + def test_different_stddev_params(self): + """Different nbdevup/nbdevdn should widen/narrow bands.""" + from ml4t.engineer.features.volatility.bollinger_bands import bollinger_bands + + np.random.seed(42) + close = 100 + np.cumsum(np.random.randn(100) * 0.5) + + u1, m1, l1 = bollinger_bands(close, period=20, nbdevup=1.0, nbdevdn=1.0) + u2, m2, l2 = bollinger_bands(close, period=20, nbdevup=3.0, nbdevdn=3.0) + + mask = ~(np.isnan(u1) | np.isnan(u2)) + # Wider bands with larger nbdev + assert np.all(u2[mask] >= u1[mask]) + assert np.all(l2[mask] <= l1[mask]) + # Middle band should be the same + assert np.allclose(m1[mask], m2[mask]) + + def test_constant_prices_collapse(self): + """Constant prices => bands collapse to price level.""" + from ml4t.engineer.features.volatility.bollinger_bands import bollinger_bands + + close = np.array([100.0] * 50) + upper, middle, lower = bollinger_bands(close, period=20) + + mask = ~np.isnan(upper) + assert np.allclose(upper[mask], 100.0) + assert np.allclose(middle[mask], 100.0) + assert np.allclose(lower[mask], 100.0) + + def test_output_column_count(self): + """Should return exactly 3 arrays.""" + from ml4t.engineer.features.volatility.bollinger_bands import bollinger_bands + + close = np.random.randn(50).cumsum() + 100 + result = bollinger_bands(close, period=10) + assert isinstance(result, tuple) + assert len(result) == 3 + + def test_different_periods(self): + """Longer period produces smoother bands.""" + from ml4t.engineer.features.volatility.bollinger_bands import bollinger_bands + + np.random.seed(42) + close = 100 + np.cumsum(np.random.randn(200) * 0.5) + + u10, m10, l10 = bollinger_bands(close, period=10) + u50, m50, l50 = bollinger_bands(close, period=50) + + # Longer period has fewer NaN at start + assert np.sum(np.isnan(m10)) < np.sum(np.isnan(m50)) + + # Both should have valid values + assert np.sum(~np.isnan(u10)) > 0 + assert np.sum(~np.isnan(u50)) > 0 + + +# --------------------------------------------------------------------------- +# Cross-estimator comparisons +# --------------------------------------------------------------------------- + + +class TestVolatilityEstimatorRelationships: + """Test relationships between different volatility estimators.""" + + def test_estimators_agree_on_magnitude(self, ohlcv_df): + """All OHLC estimators should agree on approximate vol level.""" + from ml4t.engineer.features.volatility.garman_klass_volatility import ( + garman_klass_volatility, + ) + from ml4t.engineer.features.volatility.parkinson_volatility import parkinson_volatility + from ml4t.engineer.features.volatility.rogers_satchell_volatility import ( + rogers_satchell_volatility, + ) + from ml4t.engineer.features.volatility.yang_zhang_volatility import yang_zhang_volatility + + period = 20 + pk = ( + ohlcv_df.select(parkinson_volatility("high", "low", period=period).alias("v"))["v"] + .drop_nulls() + .drop_nans() + .mean() + ) + gk = ( + ohlcv_df.select( + garman_klass_volatility("open", "high", "low", "close", period=period).alias("v") + )["v"] + .drop_nulls() + .drop_nans() + .mean() + ) + rs = ( + ohlcv_df.select( + rogers_satchell_volatility("open", "high", "low", "close", period=period).alias("v") + )["v"] + .drop_nulls() + .drop_nans() + .mean() + ) + yz = ( + ohlcv_df.select( + yang_zhang_volatility("open", "high", "low", "close", period=period).alias("v") + )["v"] + .drop_nulls() + .drop_nans() + .mean() + ) + + estimates = [pk, gk, rs, yz] + # All should be in the same order of magnitude + min_est = min(estimates) + max_est = max(estimates) + assert max_est / min_est < 10 # Within 10x of each other diff --git a/tests/test_ad.py b/tests/test_ad.py index 55cda74..f2e897f 100644 --- a/tests/test_ad.py +++ b/tests/test_ad.py @@ -178,6 +178,7 @@ def test_ad_crypto_accuracy(self, crypto_data_small): # Should match closely on real data assert_allclose(result, expected, rtol=1e-10, equal_nan=True) + @pytest.mark.perf @pytest.mark.benchmark def test_ad_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark AD performance using real crypto data.""" diff --git a/tests/test_adosc.py b/tests/test_adosc.py index 7c26d6e..237bcbd 100644 --- a/tests/test_adosc.py +++ b/tests/test_adosc.py @@ -210,6 +210,7 @@ def test_adosc_crypto_accuracy(self, crypto_data_small): f"ADOSC (crypto, fast={fast}, slow={slow})", ) + @pytest.mark.perf @pytest.mark.benchmark def test_adosc_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark ADOSC performance using real crypto data.""" diff --git a/tests/test_adxr.py b/tests/test_adxr.py index 7179d3e..bf48faf 100644 --- a/tests/test_adxr.py +++ b/tests/test_adxr.py @@ -208,6 +208,7 @@ def test_adxr_trending_vs_ranging(self): assert avg_trend > avg_range, "ADXR should be higher in trending markets" + @pytest.mark.perf @pytest.mark.benchmark def test_adxr_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark ADXR performance.""" diff --git a/tests/test_avgdev.py b/tests/test_avgdev.py index 4e70911..582dce8 100644 --- a/tests/test_avgdev.py +++ b/tests/test_avgdev.py @@ -185,6 +185,7 @@ def test_avgdev_crypto_accuracy(self, crypto_data_small): err_msg=f"AVGDEV mismatch on crypto data for period {period}", ) + @pytest.mark.perf @pytest.mark.benchmark def test_avgdev_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark AVGDEV performance.""" diff --git a/tests/test_avgprice.py b/tests/test_avgprice.py index d4071ab..e257ca9 100644 --- a/tests/test_avgprice.py +++ b/tests/test_avgprice.py @@ -163,6 +163,7 @@ def test_avgprice_crypto_accuracy(self, crypto_data_small): # Should match exactly on real data assert_allclose(result, expected, rtol=1e-10, equal_nan=True) + @pytest.mark.perf @pytest.mark.benchmark def test_avgprice_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark AVGPRICE performance using real crypto data.""" diff --git a/tests/test_bars.py b/tests/test_bars.py index 7035717..e143a7f 100644 --- a/tests/test_bars.py +++ b/tests/test_bars.py @@ -175,7 +175,6 @@ def test_basic_imbalance_bars(self, tick_data): """Test basic imbalance bar generation.""" sampler = ImbalanceBarSampler( expected_ticks_per_bar=20, - initial_expectation=1000, ) bars = sampler.sample(tick_data) @@ -219,7 +218,6 @@ def test_imbalance_bars_triggering(self): sampler = ImbalanceBarSampler( expected_ticks_per_bar=20, - initial_expectation=500, ) bars = sampler.sample(data) @@ -244,7 +242,6 @@ def test_expected_imbalance_updates(self): sampler = ImbalanceBarSampler( expected_ticks_per_bar=20, - initial_expectation=1500, # 15 ticks * 100 volume alpha=0.3, ) bars = sampler.sample(data) @@ -311,7 +308,7 @@ def test_empty_data(self): TickBarSampler(10), VolumeBarSampler(1000), DollarBarSampler(10000), - ImbalanceBarSampler(20, 100), + ImbalanceBarSampler(20), ] for sampler in samplers: @@ -362,51 +359,12 @@ def test_all_zero_volume(self): assert len(result) == 0 -class TestBarSamplerIntegration: - """Test integration with pipeline.""" - - # @pytest.mark.skip(reason="Pipeline integration with bar samplers has column resolution issues") - def test_pipeline_integration(self, tick_data): - """Test bar samplers in pipeline.""" - from ml4t.engineer.pipeline import Pipeline - - # Create pipeline that generates different bar types - pipeline = Pipeline( - steps=[ - # Generate volume bars - ("volume_bars", lambda df: VolumeBarSampler(1000).sample(df)), - # Add returns - ( - "returns", - lambda df: df.with_columns(returns=pl.col("close").pct_change()), - ), - # Add volatility - ( - "volatility", - lambda df: df.with_columns( - volatility=pl.col("returns").rolling_std(window_size=5), - ), - ), - ], - ) - - result = pipeline.run(tick_data) - - # Should have volume bar columns plus added features - assert "returns" in result.columns - assert "volatility" in result.columns - assert "volume" in result.columns - - # Volume should meet threshold - assert (result["volume"][:-1] >= 1000).all() - - class TestTickRunBarSampler: """Test tick run bar sampling.""" def test_basic_tick_run_bars(self, tick_data): """Test basic tick run bar generation.""" - sampler = TickRunBarSampler(expected_ticks_per_bar=100, initial_run_expectation=10) + sampler = TickRunBarSampler(expected_ticks_per_bar=100) bars = sampler.sample(tick_data) # Should have bars when runs exceed expectation @@ -483,7 +441,7 @@ def test_tick_run_direction_change(self): }, ) - sampler = TickRunBarSampler(expected_ticks_per_bar=20, initial_run_expectation=3) + sampler = TickRunBarSampler(expected_ticks_per_bar=20) bars = sampler.sample(data) # Should create bars when runs reach threshold @@ -496,7 +454,7 @@ class TestVolumeRunBarSampler: def test_basic_volume_run_bars(self, tick_data): """Test basic volume run bar generation.""" # Use lower initial expectation for random data - sampler = VolumeRunBarSampler(expected_ticks_per_bar=100, initial_run_expectation=500.0) + sampler = VolumeRunBarSampler(expected_ticks_per_bar=100) bars = sampler.sample(tick_data) # Should have bars @@ -547,7 +505,7 @@ def test_volume_run_with_large_volumes(self): }, ) - sampler = VolumeRunBarSampler(expected_ticks_per_bar=10, initial_run_expectation=300) + sampler = VolumeRunBarSampler(expected_ticks_per_bar=10) bars = sampler.sample(data) # Should create bars when run volumes exceed threshold @@ -561,7 +519,7 @@ class TestDollarRunBarSampler: def test_basic_dollar_run_bars(self, tick_data): """Test basic dollar run bar generation.""" # Use lower initial expectation for random data - sampler = DollarRunBarSampler(expected_ticks_per_bar=100, initial_run_expectation=5000.0) + sampler = DollarRunBarSampler(expected_ticks_per_bar=100) bars = sampler.sample(tick_data) # Should have bars @@ -654,7 +612,7 @@ class TestRunBarsComparison: def test_run_bars_create_fewer_bars_than_standard(self, tick_data): """Test that run bars create fewer but more informative bars.""" tick_sampler = TickBarSampler(ticks_per_bar=100) - tick_run_sampler = TickRunBarSampler(expected_ticks_per_bar=100, initial_run_expectation=10) + tick_run_sampler = TickRunBarSampler(expected_ticks_per_bar=100) standard_bars = tick_sampler.sample(tick_data) run_bars = tick_run_sampler.sample(tick_data) @@ -666,9 +624,9 @@ def test_run_bars_create_fewer_bars_than_standard(self, tick_data): def test_all_run_bar_types_produce_valid_output(self, tick_data): """Test that all three run bar types produce valid OHLCV data.""" # Use appropriate initial expectations for random data - tick_run = TickRunBarSampler(expected_ticks_per_bar=100, initial_run_expectation=10) - volume_run = VolumeRunBarSampler(expected_ticks_per_bar=100, initial_run_expectation=500.0) - dollar_run = DollarRunBarSampler(expected_ticks_per_bar=100, initial_run_expectation=5000.0) + tick_run = TickRunBarSampler(expected_ticks_per_bar=100) + volume_run = VolumeRunBarSampler(expected_ticks_per_bar=100) + dollar_run = DollarRunBarSampler(expected_ticks_per_bar=100) tick_bars = tick_run.sample(tick_data) volume_bars = volume_run.sample(tick_data) diff --git a/tests/test_bars_specialized.py b/tests/test_bars_specialized.py index 3df3960..130ab7e 100644 --- a/tests/test_bars_specialized.py +++ b/tests/test_bars_specialized.py @@ -352,11 +352,9 @@ def test_init_validation(self) -> None: # Valid sampler = ImbalanceBarSampler( expected_ticks_per_bar=100, - initial_expectation=1000, alpha=0.1, ) assert sampler.expected_ticks_per_bar == 100 - assert sampler.initial_expectation == 1000 assert sampler.alpha == 0.1 # Invalid expected_ticks_per_bar @@ -416,7 +414,6 @@ def test_adaptive_threshold(self) -> None: tick_data = generate_tick_data(n_ticks=1000, seed=42) sampler = ImbalanceBarSampler( expected_ticks_per_bar=50, - initial_expectation=1000, alpha=0.1, ) @@ -428,25 +425,17 @@ def test_adaptive_threshold(self) -> None: # Check that expectations change assert np.std(expectations) > 0 - def test_initial_expectation_estimation(self) -> None: - """Test automatic estimation of initial expectation.""" + def test_dynamic_threshold_estimation(self) -> None: + """Test that AFML threshold is computed dynamically from data.""" tick_data = generate_tick_data(n_ticks=500, seed=42) - # Don't provide initial_expectation - sampler = ImbalanceBarSampler( - expected_ticks_per_bar=100, - initial_expectation=None, # Will be estimated - ) + sampler = ImbalanceBarSampler(expected_ticks_per_bar=100) bars = sampler.sample(tick_data) - # Should still produce bars + # Should produce bars with dynamically computed thresholds assert len(bars) > 0 - # NOTE: initial_expectation is now computed dynamically per AFML methodology - # and is not stored on the sampler instance. The threshold adapts based on - # the data, so we just verify bars are produced successfully. - # ============================================================================= # Data Validation Tests diff --git a/tests/test_cmo.py b/tests/test_cmo.py index 49301de..718b63a 100644 --- a/tests/test_cmo.py +++ b/tests/test_cmo.py @@ -186,6 +186,7 @@ def test_cmo_crypto_accuracy(self, crypto_data_small): err_msg=f"CMO mismatch on crypto data for period {period}", ) + @pytest.mark.perf @pytest.mark.benchmark def test_cmo_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark CMO performance.""" diff --git a/tests/test_config_system.py b/tests/test_config_system.py index 317266a..a12d80d 100644 --- a/tests/test_config_system.py +++ b/tests/test_config_system.py @@ -422,57 +422,6 @@ def test_verbose_flag(self) -> None: # ============================================================================= -class TestFeatureConfigValidators: - """Tests for feature_config.py validators.""" - - def test_stationarity_config_at_least_one_test(self) -> None: - """Test that StationarityConfig requires at least one test enabled.""" - from ml4t.engineer.config.feature_config import StationarityConfig - - # Valid: at least one test enabled - config = StationarityConfig(adf_enabled=True, kpss_enabled=False, pp_enabled=False) - assert config.adf_enabled is True - - config = StationarityConfig(adf_enabled=False, kpss_enabled=True, pp_enabled=False) - assert config.kpss_enabled is True - - config = StationarityConfig(adf_enabled=False, kpss_enabled=False, pp_enabled=True) - assert config.pp_enabled is True - - def test_stationarity_config_none_enabled_raises(self) -> None: - """Test that StationarityConfig raises if no tests enabled.""" - from ml4t.engineer.config.feature_config import StationarityConfig - - with pytest.raises(ValidationError, match="At least one stationarity test"): - StationarityConfig(adf_enabled=False, kpss_enabled=False, pp_enabled=False) - - def test_volatility_config_window_sizes_validation(self) -> None: - """Test VolatilityConfig window_sizes validator.""" - from ml4t.engineer.config.feature_config import VolatilityConfig - - # Valid window sizes - config = VolatilityConfig(window_sizes=[5, 10, 20]) - assert config.window_sizes == [5, 10, 20] - - # Window sizes get sorted - config = VolatilityConfig(window_sizes=[20, 5, 10]) - assert config.window_sizes == [5, 10, 20] - - def test_volatility_config_empty_window_sizes_raises(self) -> None: - """Test that empty window_sizes raises error.""" - from ml4t.engineer.config.feature_config import VolatilityConfig - - with pytest.raises(ValidationError, match="at least one window size"): - VolatilityConfig(window_sizes=[]) - - def test_volatility_config_invalid_window_size_raises(self) -> None: - """Test that window size < 2 raises error.""" - from ml4t.engineer.config.feature_config import VolatilityConfig - - with pytest.raises(ValidationError, match="must be >= 2"): - VolatilityConfig(window_sizes=[1, 5, 10]) - - class TestConfigIntegration: """Integration tests for config system.""" diff --git a/tests/test_directional_indicators.py b/tests/test_directional_indicators.py index 5d6b6a2..0776036 100644 --- a/tests/test_directional_indicators.py +++ b/tests/test_directional_indicators.py @@ -272,6 +272,7 @@ def test_directional_crypto_accuracy(self, crypto_data_small): rtol=1e-10, ) + @pytest.mark.perf @pytest.mark.benchmark def test_directional_performance( self, diff --git a/tests/test_dm_indicators.py b/tests/test_dm_indicators.py index 000806b..17b58df 100644 --- a/tests/test_dm_indicators.py +++ b/tests/test_dm_indicators.py @@ -237,6 +237,7 @@ def test_dm_calculation_details(self): assert_indicator_match(plus_dm_result, expected_plus, "PLUS_DM calc", rtol=1e-10) assert_indicator_match(minus_dm_result, expected_minus, "MINUS_DM calc", rtol=1e-10) + @pytest.mark.perf @pytest.mark.benchmark def test_dm_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark DM indicators performance.""" diff --git a/tests/test_fdiff.py b/tests/test_fdiff.py index 386191e..cf5152e 100644 --- a/tests/test_fdiff.py +++ b/tests/test_fdiff.py @@ -232,44 +232,6 @@ def test_diagnostics_different_d_values(self): assert abs(diag_03["correlation"]) > abs(diag_07["correlation"]) -class TestPipelineIntegration: - """Test integration with pipeline API.""" - - def test_ffdiff_in_pipeline(self): - """FFD should work in pipeline context.""" - from ml4t.engineer.pipeline import Pipeline - - # Create test data - np.random.seed(42) - data = pl.DataFrame( - { - "timestamp": pl.datetime_range( - start=pl.datetime(2024, 1, 1), - end=pl.datetime(2024, 1, 10), - interval="1h", - eager=True, - )[:200], - "close": np.random.randn(200).cumsum() + 100, - }, - ) - - # Create pipeline - pipeline = Pipeline( - steps=[ - ( - "returns", - lambda df: df.with_columns(returns=pl.col("close").pct_change()), - ), - ("ffd", lambda df: df.with_columns(close_ffd=ffdiff("close", d=0.5))), - ], - ) - - result = pipeline.run(data) - - assert "close_ffd" in result.columns - assert len(result) == len(data) - - class TestEdgeCases: """Test edge cases and error handling.""" diff --git a/tests/test_imi.py b/tests/test_imi.py index 786ac9a..77a359d 100644 --- a/tests/test_imi.py +++ b/tests/test_imi.py @@ -195,6 +195,7 @@ def test_imi_calculation_details(self): assert_indicator_match(imi_result, expected, "IMI calculation", rtol=1e-6) + @pytest.mark.perf @pytest.mark.benchmark def test_imi_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark IMI performance.""" diff --git a/tests/test_integration_pipeline.py b/tests/test_integration_pipeline.py deleted file mode 100644 index e0386c9..0000000 --- a/tests/test_integration_pipeline.py +++ /dev/null @@ -1,430 +0,0 @@ -"""Integration tests: OHLCV → features → labeling → preprocessing → MLDatasetBuilder. - -These tests verify the full pipeline a book reader would use, end-to-end. -No mocking — real computations with synthetic but realistic data. -""" - -from datetime import datetime, timedelta - -import numpy as np -import polars as pl -import pytest - -from ml4t.engineer.config import LabelingConfig, PreprocessingConfig -from ml4t.engineer.dataset import MLDatasetBuilder -from ml4t.engineer.labeling import ( - atr_triple_barrier_labels, - fixed_time_horizon_labels, - triple_barrier_labels, -) -from ml4t.engineer.preprocessing import RobustScaler, StandardScaler - - -@pytest.fixture -def ohlcv_data(): - """Realistic synthetic OHLCV data for a single asset (200 bars).""" - np.random.seed(42) - n = 200 - timestamps = [datetime(2024, 1, 1) + timedelta(hours=i) for i in range(n)] - - # Generate correlated OHLCV with realistic structure - close = 100.0 + np.cumsum(np.random.randn(n) * 0.5) - high = close + np.abs(np.random.randn(n) * 0.3) - low = close - np.abs(np.random.randn(n) * 0.3) - open_ = close + np.random.randn(n) * 0.1 - volume = np.abs(np.random.randn(n) * 1000 + 5000) - - return pl.DataFrame( - { - "timestamp": timestamps, - "open": open_.tolist(), - "high": high.tolist(), - "low": low.tolist(), - "close": close.tolist(), - "volume": volume.tolist(), - } - ) - - -@pytest.fixture -def panel_data(): - """Multi-asset OHLCV data for panel testing (2 assets, 100 bars each).""" - np.random.seed(123) - n = 100 - - frames = [] - for symbol in ["AAPL", "MSFT"]: - timestamps = [datetime(2024, 1, 1) + timedelta(hours=i) for i in range(n)] - close = 100.0 + np.cumsum(np.random.randn(n) * 0.3) - high = close + np.abs(np.random.randn(n) * 0.2) - low = close - np.abs(np.random.randn(n) * 0.2) - open_ = close + np.random.randn(n) * 0.05 - volume = np.abs(np.random.randn(n) * 500 + 3000) - - frames.append( - pl.DataFrame( - { - "timestamp": timestamps, - "symbol": [symbol] * n, - "open": open_.tolist(), - "high": high.tolist(), - "low": low.tolist(), - "close": close.tolist(), - "volume": volume.tolist(), - } - ) - ) - return pl.concat(frames) - - -class TestFixedHorizonPipeline: - """Test: OHLCV → fixed_time_horizon_labels → scaler → numpy.""" - - def test_bar_based_labels_to_numpy(self, ohlcv_data): - # Label - labeled = fixed_time_horizon_labels( - ohlcv_data, - horizon=5, - method="returns", - price_col="close", - group_col=[], - ) - assert "label_return_5p" in labeled.columns - assert labeled.shape[0] == ohlcv_data.shape[0] - - # The last 5 rows should be null (no future data) - label_col = labeled["label_return_5p"] - assert label_col[-1] is None - - def test_time_based_labels(self, ohlcv_data): - labeled = fixed_time_horizon_labels( - ohlcv_data, - horizon="2h", - method="returns", - price_col="close", - timestamp_col="timestamp", - group_col=[], - ) - assert any("label_return" in c for c in labeled.columns) - - def test_binary_labels(self, ohlcv_data): - labeled = fixed_time_horizon_labels( - ohlcv_data, - horizon=1, - method="binary", - price_col="close", - group_col=[], - ) - label_col = "label_direction_1p" - assert label_col in labeled.columns - # Non-null values should be in {-1, 0, 1} - non_null = labeled.filter(pl.col(label_col).is_not_null())[label_col] - assert set(non_null.to_list()).issubset({-1, 0, 1}) - - def test_panel_data_labels(self, panel_data): - labeled = fixed_time_horizon_labels( - panel_data, - horizon=5, - method="returns", - price_col="close", - group_col="symbol", - timestamp_col="timestamp", - ) - assert labeled.shape[0] == panel_data.shape[0] - # Both assets should have labels - per_asset = labeled.group_by("symbol").agg( - pl.col("label_return_5p").is_not_null().sum().alias("non_null_count") - ) - for row in per_asset.iter_rows(named=True): - assert row["non_null_count"] > 0 - - -class TestTripleBarrierPipeline: - """Test: OHLCV → triple_barrier_labels → preprocessing → numpy.""" - - def test_basic_labeling(self, ohlcv_data): - config = LabelingConfig.triple_barrier( - upper_barrier=0.02, - lower_barrier=0.01, - max_holding_period=20, - ) - labeled = triple_barrier_labels( - ohlcv_data, - config=config, - price_col="close", - timestamp_col="timestamp", - group_col=[], - ) - assert "label" in labeled.columns - assert "label_return" in labeled.columns - assert "barrier_hit" in labeled.columns - - # Labels should be in {-1, 0, 1} - non_null = labeled.filter(pl.col("label").is_not_null())["label"] - assert set(non_null.to_list()).issubset({-1, 0, 1}) - - def test_atr_labeling(self, ohlcv_data): - labeled = atr_triple_barrier_labels( - ohlcv_data, - atr_tp_multiple=2.0, - atr_sl_multiple=1.0, - atr_period=14, - max_holding_bars=30, - price_col="close", - timestamp_col="timestamp", - group_col=[], - ) - assert "label" in labeled.columns - assert "atr" in labeled.columns - assert "upper_barrier_distance" in labeled.columns - assert "lower_barrier_distance" in labeled.columns - - def test_panel_triple_barrier(self, panel_data): - config = LabelingConfig.triple_barrier( - upper_barrier=0.02, - lower_barrier=0.01, - max_holding_period=15, - ) - labeled = triple_barrier_labels( - panel_data, - config=config, - price_col="close", - timestamp_col="timestamp", - group_col="symbol", - ) - assert labeled.shape[0] == panel_data.shape[0] - # Both assets should have labels - per_asset = labeled.group_by("symbol").agg( - pl.col("label").is_not_null().sum().alias("label_count") - ) - for row in per_asset.iter_rows(named=True): - assert row["label_count"] > 0 - - -class TestPreprocessingPipeline: - """Test: labeled data → scaler → numpy arrays.""" - - def test_standard_scaler_fit_transform(self, ohlcv_data): - scaler = StandardScaler(columns=["close", "volume"]) - scaled = scaler.fit_transform(ohlcv_data) - - # Scaled columns should have ~0 mean and ~1 std - close_mean = scaled["close"].mean() - close_std = scaled["close"].std() - assert abs(close_mean) < 0.01 - assert abs(close_std - 1.0) < 0.01 - - # Non-scaled columns should be unchanged - assert scaled["timestamp"].to_list() == ohlcv_data["timestamp"].to_list() - assert scaled["open"].to_list() == ohlcv_data["open"].to_list() - - def test_robust_scaler(self, ohlcv_data): - scaler = RobustScaler(columns=["close", "volume"]) - scaled = scaler.fit_transform(ohlcv_data) - assert scaled.shape == ohlcv_data.shape - - def test_scaler_from_config(self, ohlcv_data): - config = PreprocessingConfig.robust(quantile_range=(10.0, 90.0)) - scaler = config.create_scaler() - assert scaler is not None - scaled = scaler.fit_transform(ohlcv_data) - assert scaled.shape == ohlcv_data.shape - - def test_scaler_preserves_column_order(self, ohlcv_data): - scaler = StandardScaler(columns=["volume", "close"]) - scaled = scaler.fit_transform(ohlcv_data) - assert scaled.columns == ohlcv_data.columns - - -class TestMLDatasetBuilder: - """Test: full pipeline → MLDatasetBuilder → numpy arrays.""" - - def test_basic_dataset_build(self, ohlcv_data): - # Add a simple feature and label - data = ohlcv_data.with_columns( - (pl.col("close").pct_change()).alias("returns"), - (pl.col("close").shift(-1) > pl.col("close")).cast(pl.Int8).alias("label"), - ) - clean = data.drop_nulls(subset=["returns", "label"]) - - features = clean.select(["close", "volume", "returns"]) - labels = clean["label"] - - builder = MLDatasetBuilder(features=features, labels=labels) - - # train_test_split returns Polars DataFrames/Series - X_train, X_test, y_train, y_test = builder.train_test_split(train_size=0.75) - assert isinstance(X_train, pl.DataFrame) - assert isinstance(y_train, pl.Series) - assert X_train.shape[1] == 3 # 3 features - assert len(X_train) + len(X_test) == len(clean) - - def test_to_numpy(self, ohlcv_data): - """Test conversion to numpy arrays.""" - data = ohlcv_data.with_columns( - (pl.col("close").pct_change()).alias("returns"), - (pl.col("close").shift(-1) > pl.col("close")).cast(pl.Int8).alias("label"), - ) - clean = data.drop_nulls(subset=["returns", "label"]) - - features = clean.select(["close", "volume", "returns"]) - labels = clean["label"] - - builder = MLDatasetBuilder(features=features, labels=labels) - X, y = builder.to_numpy() - assert isinstance(X, np.ndarray) - assert isinstance(y, np.ndarray) - assert X.shape == (len(clean), 3) - - def test_dataset_with_scaler(self, ohlcv_data): - data = ohlcv_data.with_columns( - (pl.col("close").pct_change()).alias("returns"), - (pl.col("close").shift(-5) > pl.col("close")).cast(pl.Int8).alias("label"), - ) - clean = data.drop_nulls(subset=["returns", "label"]) - - features = clean.select(["close", "volume", "returns"]) - labels = clean["label"] - - builder = MLDatasetBuilder(features=features, labels=labels) - builder.set_scaler(StandardScaler(columns=["close", "volume"])) - - X_train, X_test, y_train, y_test = builder.train_test_split(train_size=0.75) - assert X_train.shape[1] == 3 - - def test_dataset_dates(self, ohlcv_data): - """Test that dates are preserved through the builder.""" - data = ohlcv_data.with_columns( - (pl.col("close").pct_change()).alias("returns"), - (pl.col("close").shift(-1) > pl.col("close")).cast(pl.Int8).alias("label"), - ) - clean = data.drop_nulls(subset=["returns", "label"]) - - features = clean.select(["close", "volume", "returns"]) - labels = clean["label"] - dates = clean["timestamp"] - - builder = MLDatasetBuilder(features=features, labels=labels, dates=dates) - assert builder.dates is not None - assert len(builder.dates) == len(features) - - -class TestEndToEndPipeline: - """Full end-to-end: OHLCV → label → preprocess → dataset → numpy.""" - - def test_full_pipeline_fixed_horizon(self, ohlcv_data): - """Complete pipeline a book reader would follow.""" - # Step 1: Label with fixed horizon - labeled = fixed_time_horizon_labels( - ohlcv_data, - horizon=5, - method="returns", - price_col="close", - group_col=[], - ) - assert "label_return_5p" in labeled.columns - - # Step 2: Add features - enriched = labeled.with_columns( - (pl.col("close").pct_change()).alias("returns_1"), - (pl.col("close").pct_change(5)).alias("returns_5"), - ((pl.col("high") - pl.col("low")) / pl.col("close")).alias("hl_range"), - ) - - # Step 3: Clean nulls and build dataset - feature_cols = ["returns_1", "returns_5", "hl_range"] - label_col = "label_return_5p" - clean = enriched.drop_nulls(subset=feature_cols + [label_col]) - - features = clean.select(feature_cols) - labels = clean[label_col] - - builder = MLDatasetBuilder(features=features, labels=labels) - X_train, X_test, y_train, y_test = builder.train_test_split(train_size=0.75) - - assert isinstance(X_train, pl.DataFrame) - assert X_train.shape[1] == 3 - assert X_train.shape[0] > 0 - - # Verify numpy conversion works - X_np, y_np = builder.to_numpy() - assert isinstance(X_np, np.ndarray) - assert X_np.shape[1] == 3 - - def test_full_pipeline_triple_barrier(self, ohlcv_data): - """Triple barrier → features → scaler → dataset.""" - # Step 1: Triple barrier labeling - config = LabelingConfig.triple_barrier( - upper_barrier=0.02, - lower_barrier=0.01, - max_holding_period=20, - ) - labeled = triple_barrier_labels( - ohlcv_data, - config=config, - price_col="close", - timestamp_col="timestamp", - group_col=[], - ) - - # Step 2: Add features - enriched = labeled.with_columns( - (pl.col("close").pct_change()).alias("returns"), - ((pl.col("high") - pl.col("low")) / pl.col("close")).alias("range_pct"), - (pl.col("volume") / pl.col("volume").rolling_mean(20)).alias("volume_ratio"), - ) - - # Step 3: Clean and build dataset - feature_cols = ["returns", "range_pct", "volume_ratio"] - clean = enriched.drop_nulls(subset=feature_cols + ["label"]) - - features = clean.select(feature_cols) - labels = clean["label"] - - builder = MLDatasetBuilder(features=features, labels=labels) - builder.set_scaler(StandardScaler(columns=feature_cols)) - - X_train, X_test, y_train, y_test = builder.train_test_split(train_size=0.75) - assert X_train.shape[1] == 3 - assert X_train.shape[0] > 0 - # Labels should be in {-1, 0, 1} - assert set(y_train.to_numpy().astype(int)).issubset({-1, 0, 1}) - - -class TestNaNValidation: - """Test that NaN validation catches bad data early.""" - - def test_triple_barrier_rejects_nan_prices(self, ohlcv_data): - from ml4t.engineer.core.exceptions import DataValidationError - - bad_data = ohlcv_data.with_columns( - pl.when(pl.col("close").is_first_distinct()) - .then(float("nan")) - .otherwise(pl.col("close")) - .alias("close") - ) - config = LabelingConfig.triple_barrier( - upper_barrier=0.02, - lower_barrier=0.01, - max_holding_period=20, - ) - with pytest.raises(DataValidationError, match="null/NaN"): - triple_barrier_labels( - bad_data, - config=config, - price_col="close", - timestamp_col="timestamp", - group_col=[], - ) - - def test_fixed_horizon_rejects_nan_prices(self, ohlcv_data): - from ml4t.engineer.core.exceptions import DataValidationError - - bad_data = ohlcv_data.with_columns(pl.lit(None, dtype=pl.Float64).alias("close")) - with pytest.raises(DataValidationError, match="null/NaN"): - fixed_time_horizon_labels( - bad_data, - horizon=5, - price_col="close", - group_col=[], - ) diff --git a/tests/test_kama.py b/tests/test_kama.py index 0414974..da0c2f5 100644 --- a/tests/test_kama.py +++ b/tests/test_kama.py @@ -162,6 +162,7 @@ def test_kama_crypto_accuracy(self, crypto_data_small): err_msg=f"KAMA mismatch on crypto data for period {period}", ) + @pytest.mark.perf @pytest.mark.benchmark def test_kama_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark KAMA performance.""" diff --git a/tests/test_labeling.py b/tests/test_labeling.py index 318a753..e8cbed7 100644 --- a/tests/test_labeling.py +++ b/tests/test_labeling.py @@ -1,7 +1,5 @@ """Tests for the generalized labeling module.""" -import importlib -import sys import warnings from datetime import datetime, timedelta @@ -14,53 +12,6 @@ from ml4t.engineer.labeling import triple_barrier_labels -class TestRemovedCoreModule: - """Test explicit error messaging for removed legacy module paths.""" - - def test_labeling_core_import_error_is_actionable(self): - """Importing removed labeling.core should raise a clear migration error.""" - sys.modules.pop("ml4t.engineer.labeling.core", None) - with pytest.raises(ImportError, match="labeling.core has been removed"): - importlib.import_module("ml4t.engineer.labeling.core") - - def test_labeling_barrier_config_export_is_actionable(self): - """Importing removed labeling.BarrierConfig should raise migration guidance.""" - with pytest.raises(ImportError, match="BarrierConfig has been removed"): - from ml4t.engineer.labeling import BarrierConfig # noqa: F401 - - def test_labeling_barriers_module_import_error_is_actionable(self): - """Importing removed labeling.barriers should raise migration guidance.""" - sys.modules.pop("ml4t.engineer.labeling.barriers", None) - with pytest.raises(ImportError, match="labeling.barriers has been removed"): - importlib.import_module("ml4t.engineer.labeling.barriers") - - def test_labeling_barrier_utils_module_import_error_is_actionable(self): - """Importing removed labeling.barrier_utils should raise migration guidance.""" - sys.modules.pop("ml4t.engineer.labeling.barrier_utils", None) - with pytest.raises(ImportError, match="labeling.barrier_utils has been removed"): - importlib.import_module("ml4t.engineer.labeling.barrier_utils") - - def test_config_alias_import_error_is_actionable(self): - """Importing removed config alias should raise migration guidance.""" - with pytest.raises(ImportError, match="BarrierLabelingConfig has been removed"): - from ml4t.engineer.config import BarrierLabelingConfig # noqa: F401 - - def test_config_labeling_alias_import_error_is_actionable(self): - """Importing removed config.labeling alias should raise migration guidance.""" - with pytest.raises(ImportError, match="BarrierLabelingConfig has been removed"): - from ml4t.engineer.config.labeling import BarrierLabelingConfig # noqa: F401 - - def test_non_labeling_config_input_raises_actionable_error(self): - """Passing non-LabelingConfig should fail with migration guidance.""" - - class LegacyBarrierConfig: - pass - - df = pl.DataFrame({"timestamp": [datetime(2024, 1, 1)], "close": [100.0]}) - with pytest.raises(TypeError, match="Legacy BarrierConfig inputs are no longer supported"): - triple_barrier_labels(df, config=LegacyBarrierConfig()) # type: ignore[arg-type] - - class TestLabelingConfig: """Test labeling configuration.""" @@ -914,6 +865,7 @@ def _create_benchmark_data(self, n_bars: int) -> pl.DataFrame: } ) + @pytest.mark.perf @pytest.mark.parametrize("n_bars", [10_000, 50_000, 100_000]) def test_duration_overhead(self, benchmark, n_bars): """Benchmark duration calculation overhead. @@ -958,6 +910,7 @@ def test_duration_overhead(self, benchmark, n_bars): f"({labeled_count / n_bars * 100:.1f}% label rate)" ) + @pytest.mark.perf def test_scaling_characteristics(self, benchmark): """Test how duration calculations scale with dataset size. @@ -986,6 +939,7 @@ def test_scaling_characteristics(self, benchmark): assert "label_bars" in result.columns assert "label_duration" in result.columns + @pytest.mark.perf def test_duration_computation_only(self, benchmark): """Benchmark just the duration computation overhead. @@ -1031,6 +985,7 @@ def duration_operations(timestamps, event_indices, label_indices): print(f"\n{n_events:,} duration calculations completed") + @pytest.mark.perf @pytest.mark.parametrize("max_holding_period", [10, 20, 50, 100], ids=lambda x: f"period_{x}") def test_performance_vs_holding_period(self, benchmark, max_holding_period): """Test performance impact of different holding periods. diff --git a/tests/test_linearreg.py b/tests/test_linearreg.py index bc2df95..18badbf 100644 --- a/tests/test_linearreg.py +++ b/tests/test_linearreg.py @@ -221,6 +221,7 @@ def test_linearreg_crypto_accuracy(self, crypto_data_small): err_msg=f"LINEARREG mismatch on crypto data for period {period}", ) + @pytest.mark.perf @pytest.mark.benchmark def test_linearreg_performance( self, diff --git a/tests/test_linearreg_family.py b/tests/test_linearreg_family.py index 9b74188..5bdebea 100644 --- a/tests/test_linearreg_family.py +++ b/tests/test_linearreg_family.py @@ -317,6 +317,7 @@ def test_linearreg_family_crypto_accuracy(self, crypto_data_small): err_msg=f"LINEARREG_ANGLE mismatch on crypto data for period {period}", ) + @pytest.mark.perf @pytest.mark.benchmark def test_linearreg_family_performance( self, diff --git a/tests/test_math_operators.py b/tests/test_math_operators.py index 9580d03..2e13472 100644 --- a/tests/test_math_operators.py +++ b/tests/test_math_operators.py @@ -226,6 +226,7 @@ def test_math_operators_insufficient_data(self): assert np.all(np.isnan(min_result)) assert np.all(np.isnan(sum_result)) + @pytest.mark.perf @pytest.mark.benchmark def test_math_operators_performance( self, diff --git a/tests/test_medprice.py b/tests/test_medprice.py index f77d243..5cc1314 100644 --- a/tests/test_medprice.py +++ b/tests/test_medprice.py @@ -140,6 +140,7 @@ def test_medprice_crypto_accuracy(self, crypto_data_small): # Should match exactly on real data assert_allclose(result, expected, rtol=1e-10, equal_nan=True) + @pytest.mark.perf @pytest.mark.benchmark def test_medprice_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark MEDPRICE performance using real crypto data.""" diff --git a/tests/test_midpoint.py b/tests/test_midpoint.py index 509a3a0..5fe2a28 100644 --- a/tests/test_midpoint.py +++ b/tests/test_midpoint.py @@ -184,6 +184,7 @@ def test_midpoint_crypto_accuracy(self, crypto_data_small): err_msg=f"MIDPOINT mismatch on crypto data for period {period}", ) + @pytest.mark.perf @pytest.mark.benchmark def test_midpoint_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark MIDPOINT performance.""" diff --git a/tests/test_midprice.py b/tests/test_midprice.py index 21e5fe5..92193f0 100644 --- a/tests/test_midprice.py +++ b/tests/test_midprice.py @@ -259,6 +259,7 @@ def test_midprice_crypto_accuracy(self, crypto_data_small): err_msg=f"MIDPRICE mismatch on crypto data for period {period}", ) + @pytest.mark.perf @pytest.mark.benchmark def test_midprice_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark MIDPRICE performance.""" diff --git a/tests/test_natr.py b/tests/test_natr.py index 28c36c7..859dfe1 100644 --- a/tests/test_natr.py +++ b/tests/test_natr.py @@ -178,6 +178,7 @@ def test_natr_crypto_accuracy(self, crypto_data_small): err_msg=f"NATR mismatch for period={period} on real crypto data", ) + @pytest.mark.perf @pytest.mark.benchmark def test_natr_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark NATR performance using real crypto data.""" diff --git a/tests/test_new_indicators.py b/tests/test_new_indicators.py index edf6e1f..5ec2746 100644 --- a/tests/test_new_indicators.py +++ b/tests/test_new_indicators.py @@ -45,7 +45,7 @@ def test_mom_accuracy(self, price_data): # Test different periods for period in [10, 20, 30]: talib_mom = talib.MOM(close, timeperiod=period) - our_mom = mom(close, timeperiod=period) + our_mom = mom(close, period=period) np.testing.assert_allclose( talib_mom, @@ -68,13 +68,13 @@ def test_mom_edge_cases(self): """Test MOM edge cases.""" # Small array small = np.array([10.0, 11.0, 12.0, 13.0, 14.0]) - result = mom(small, timeperiod=3) + result = mom(small, period=3) assert np.isnan(result[:3]).all() assert result[3] == 3.0 # 13 - 10 assert result[4] == 3.0 # 14 - 11 # Period larger than data - result = mom(small, timeperiod=10) + result = mom(small, period=10) assert np.isnan(result).all() @@ -224,7 +224,7 @@ def test_stddev_with_scaling(self, price_data): class TestPerformanceComparison: """Test performance of new indicators vs TA-Lib.""" - @pytest.mark.performance + @pytest.mark.perf @pytest.mark.skipif(not HAS_TALIB, reason="TA-Lib not available") def test_mom_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark MOM performance.""" @@ -266,6 +266,7 @@ def test_mom_performance(self, crypto_data, performance_threshold, warmup_jit): f"MOM performance ratio {our_time / talib_time:.1f}x exceeds threshold {threshold}x" ) + @pytest.mark.perf @pytest.mark.skipif(not HAS_TALIB, reason="TA-Lib not available") def test_obv_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark OBV performance.""" @@ -302,6 +303,7 @@ def test_obv_performance(self, crypto_data, performance_threshold, warmup_jit): threshold = performance_threshold("simple") assert our_time < talib_time * threshold + @pytest.mark.perf @pytest.mark.skipif(not HAS_TALIB, reason="TA-Lib not available") def test_ppo_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark PPO performance.""" @@ -415,6 +417,7 @@ def test_aroon_parameter_validation(self): with pytest.raises((ValueError, InvalidParameterError)): aroon(high, low, timeperiod=1) + @pytest.mark.perf @pytest.mark.skipif(not HAS_TALIB, reason="TA-Lib not available") def test_aroon_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark AROON performance.""" @@ -518,6 +521,7 @@ def test_sar_parameter_validation(self): with pytest.raises((ValueError, InvalidParameterError)): sar(high, low, acceleration=0.3, maximum=0.2) + @pytest.mark.perf @pytest.mark.skipif(not HAS_TALIB, reason="TA-Lib not available") def test_sar_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark SAR performance.""" diff --git a/tests/test_optimized_indicators.py b/tests/test_optimized_indicators.py index 12c358b..e8766e3 100644 --- a/tests/test_optimized_indicators.py +++ b/tests/test_optimized_indicators.py @@ -351,6 +351,7 @@ def benchmark_indicator( "numba_throughput": numba_throughput, } + @pytest.mark.perf def test_performance_comparison(self, large_data): """Compare performance of Polars vs Numba implementations.""" print("\n" + "=" * 60) diff --git a/tests/test_performance.py b/tests/test_performance.py index e8e3bdf..f7c84a6 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -74,6 +74,7 @@ def time_function(self, func, *args, **kwargs): end = time.perf_counter() return result, end - start + @pytest.mark.perf def test_sma_performance_small(self): """Test SMA performance on small dataset.""" # Our implementation @@ -95,6 +96,7 @@ def test_sma_performance_small(self): # Performance threshold: should complete in reasonable time assert our_time < 1.0, f"SMA too slow: {our_time:.4f}s" + @pytest.mark.perf def test_sma_performance_large(self): """Test SMA performance on large dataset.""" # Our implementation @@ -116,6 +118,7 @@ def test_sma_performance_large(self): # Performance threshold: should handle 100K rows efficiently assert our_time < 5.0, f"SMA too slow on large data: {our_time:.4f}s" + @pytest.mark.perf def test_multiple_indicators_performance(self): """Test performance of computing multiple indicators simultaneously.""" indicators = { @@ -147,6 +150,7 @@ def test_multiple_indicators_performance(self): f"Too slow: {calculations_per_second:,.0f} calculations/sec" ) + @pytest.mark.perf def test_rsi_performance_vs_talib(self): """Test RSI performance vs TA-Lib.""" if not HAS_TALIB: @@ -173,6 +177,7 @@ def test_rsi_performance_vs_talib(self): # Should be competitive with TA-Lib assert speedup > 0.5, f"Too slow compared to TA-Lib: {speedup:.2f}x" + @pytest.mark.perf def test_bollinger_bands_performance(self): """Test Bollinger Bands performance.""" _, our_time = self.time_function( @@ -201,6 +206,7 @@ def test_bollinger_bands_performance(self): # Should complete in reasonable time assert our_time < 2.0, f"Bollinger Bands too slow: {our_time:.4f}s" + @pytest.mark.perf def test_memory_efficiency(self): """Test memory efficiency with large datasets.""" # Test with lazy evaluation - should not consume excessive memory @@ -224,6 +230,7 @@ def test_memory_efficiency(self): # Should complete without memory issues assert collect_time < 10.0, f"Collection too slow: {collect_time:.4f}s" + @pytest.mark.perf def test_streaming_performance(self): """Test performance with streaming-like operations.""" # Simulate processing data in chunks @@ -272,6 +279,7 @@ def time_function(self, func): end = time.perf_counter() return result, end - start + @pytest.mark.perf def test_million_row_performance(self): """Test performance with 1 million rows.""" # Run with reduced size for reasonable test time diff --git a/tests/test_pipeline_engine.py b/tests/test_pipeline_engine.py deleted file mode 100644 index 2c1826f..0000000 --- a/tests/test_pipeline_engine.py +++ /dev/null @@ -1,366 +0,0 @@ -"""Tests for the pipeline engine DAG functionality.""" - -import polars as pl -import pytest - -from ml4t.engineer.pipeline.engine import Pipeline, PipelineStep - - -class TestPipelineEngine: - """Test the DAG-based pipeline engine.""" - - @pytest.fixture - def sample_data(self) -> pl.DataFrame: - """Create sample data for testing.""" - return pl.DataFrame( - { - "price": [ - 100.0, - 102.0, - 101.0, - 103.0, - 104.0, - 102.0, - 105.0, - 106.0, - 104.0, - 107.0, - ], - "volume": [1000, 1200, 800, 1500, 1100, 900, 1300, 1400, 1000, 1600], - }, - ) - - def test_simple_pipeline_no_dependencies(self, sample_data: pl.DataFrame) -> None: - """Test pipeline with steps that have no dependencies.""" - - def add_returns(df: pl.DataFrame) -> pl.DataFrame: - return df.with_columns(returns=pl.col("price").pct_change()) - - def add_log_returns(df: pl.DataFrame) -> pl.DataFrame: - return df.with_columns(log_returns=pl.col("price").log().diff()) - - pipeline = Pipeline( - [("returns", add_returns), ("log_returns", add_log_returns)], - ) - - result = pipeline.run(sample_data) - - # Check that both columns were added - assert "returns" in result.columns - assert "log_returns" in result.columns - assert len(result) == len(sample_data) - - def test_pipeline_with_dependencies(self, sample_data: pl.DataFrame) -> None: - """Test pipeline with step dependencies.""" - - def add_returns(df: pl.DataFrame) -> pl.DataFrame: - return df.with_columns(returns=pl.col("price").pct_change()) - - def add_volatility(df: pl.DataFrame) -> pl.DataFrame: - return df.with_columns( - volatility=pl.col("returns").rolling_std(window_size=3), - ) - - def add_sharpe(df: pl.DataFrame) -> pl.DataFrame: - return df.with_columns( - sharpe=pl.col("returns").mean() / pl.col("volatility"), - ) - - # Create pipeline with dependencies - pipeline = Pipeline( - [ - PipelineStep("returns", add_returns), - PipelineStep("volatility", add_volatility, dependencies=["returns"]), - PipelineStep( - "sharpe", - add_sharpe, - dependencies=["returns", "volatility"], - ), - ], - ) - - result = pipeline.run(sample_data) - - # Verify all columns exist - assert "returns" in result.columns - assert "volatility" in result.columns - assert "sharpe" in result.columns - - def test_execution_order_respects_dependencies( - self, - sample_data: pl.DataFrame, - ) -> None: - """Test that execution order respects dependencies.""" - execution_log = [] - - def step_a(df: pl.DataFrame) -> pl.DataFrame: - execution_log.append("A") - return df.with_columns(col_a=pl.lit(1)) - - def step_b(df: pl.DataFrame) -> pl.DataFrame: - execution_log.append("B") - return df.with_columns(col_b=pl.lit(2)) - - def step_c(df: pl.DataFrame) -> pl.DataFrame: - execution_log.append("C") - return df.with_columns(col_c=pl.lit(3)) - - def step_d(df: pl.DataFrame) -> pl.DataFrame: - execution_log.append("D") - return df.with_columns(col_d=pl.lit(4)) - - # Create pipeline: D depends on C and B, C depends on A, B depends on A - # Expected order: A, then B and C (in either order), then D - pipeline = Pipeline( - [ - PipelineStep("D", step_d, dependencies=["C", "B"]), - PipelineStep("B", step_b, dependencies=["A"]), - PipelineStep("A", step_a), - PipelineStep("C", step_c, dependencies=["A"]), - ], - ) - - pipeline.run(sample_data) - - # A must come first - assert execution_log[0] == "A" - # D must come last - assert execution_log[-1] == "D" - # B and C must come after A but before D - assert execution_log.index("B") > execution_log.index("A") - assert execution_log.index("C") > execution_log.index("A") - assert execution_log.index("B") < execution_log.index("D") - assert execution_log.index("C") < execution_log.index("D") - - def test_cycle_detection(self) -> None: - """Test that cycles are properly detected.""" - - def dummy_step(df: pl.DataFrame) -> pl.DataFrame: - return df - - # Create a cycle: A -> B -> C -> A - with pytest.raises(ValueError, match="Cycle detected in pipeline"): - Pipeline( - [ - PipelineStep("A", dummy_step, dependencies=["C"]), - PipelineStep("B", dummy_step, dependencies=["A"]), - PipelineStep("C", dummy_step, dependencies=["B"]), - ], - ) - - def test_self_dependency_cycle(self) -> None: - """Test detection of self-dependency cycles.""" - - def dummy_step(df: pl.DataFrame) -> pl.DataFrame: - return df - - with pytest.raises(ValueError, match="Cycle detected in pipeline"): - Pipeline([PipelineStep("A", dummy_step, dependencies=["A"])]) - - def test_unknown_dependency(self) -> None: - """Test error when depending on unknown step.""" - - def dummy_step(df: pl.DataFrame) -> pl.DataFrame: - return df - - with pytest.raises(ValueError, match="depends on unknown step"): - Pipeline([PipelineStep("A", dummy_step, dependencies=["unknown_step"])]) - - def test_complex_dag_ordering(self, sample_data: pl.DataFrame) -> None: - """Test complex DAG with multiple dependency levels.""" - execution_log = [] - - def make_step(name: str): - def step(df: pl.DataFrame) -> pl.DataFrame: - execution_log.append(name) - return df.with_columns(**{f"col_{name.lower()}": pl.lit(1)}) - - return step - - # Complex DAG: - # A - # / \ - # B C - # | / | \ - # D E F G - # \ | | / - # H I - # \ | - # J - - pipeline = Pipeline( - [ - PipelineStep("J", make_step("J"), dependencies=["H", "I"]), - PipelineStep("I", make_step("I"), dependencies=["F", "G"]), - PipelineStep("H", make_step("H"), dependencies=["D", "E"]), - PipelineStep("G", make_step("G"), dependencies=["C"]), - PipelineStep("F", make_step("F"), dependencies=["C"]), - PipelineStep("E", make_step("E"), dependencies=["C"]), - PipelineStep("D", make_step("D"), dependencies=["B"]), - PipelineStep("C", make_step("C"), dependencies=["A"]), - PipelineStep("B", make_step("B"), dependencies=["A"]), - PipelineStep("A", make_step("A")), - ], - ) - - result = pipeline.run(sample_data) - - # Check topological constraints - def get_position(name: str) -> int: - return execution_log.index(name) - - # A must come before all others - assert get_position("A") < get_position("B") - assert get_position("A") < get_position("C") - - # B must come before D - assert get_position("B") < get_position("D") - - # C must come before E, F, G - assert get_position("C") < get_position("E") - assert get_position("C") < get_position("F") - assert get_position("C") < get_position("G") - - # D and E must come before H - assert get_position("D") < get_position("H") - assert get_position("E") < get_position("H") - - # F and G must come before I - assert get_position("F") < get_position("I") - assert get_position("G") < get_position("I") - - # H and I must come before J - assert get_position("H") < get_position("J") - assert get_position("I") < get_position("J") - - # Verify all columns were created - expected_cols = [f"col_{name.lower()}" for name in "ABCDEFGHIJ"] - for col in expected_cols: - assert col in result.columns - - def test_add_step_method(self, sample_data: pl.DataFrame) -> None: - """Test dynamically adding steps to pipeline.""" - - def add_returns(df: pl.DataFrame) -> pl.DataFrame: - return df.with_columns(returns=pl.col("price").pct_change()) - - def add_volatility(df: pl.DataFrame) -> pl.DataFrame: - return df.with_columns( - volatility=pl.col("returns").rolling_std(window_size=3), - ) - - # Start with empty pipeline - pipeline = Pipeline([]) - - # Add steps dynamically - pipeline.add_step(("returns", add_returns)) - pipeline.add_step( - PipelineStep("volatility", add_volatility, dependencies=["returns"]), - ) - - result = pipeline.run(sample_data) - - assert "returns" in result.columns - assert "volatility" in result.columns - - def test_get_intermediate_result(self, sample_data: pl.DataFrame) -> None: - """Test accessing intermediate results.""" - - def add_returns(df: pl.DataFrame) -> pl.DataFrame: - return df.with_columns(returns=pl.col("price").pct_change()) - - def add_volatility(df: pl.DataFrame) -> pl.DataFrame: - return df.with_columns( - volatility=pl.col("returns").rolling_std(window_size=3), - ) - - pipeline = Pipeline( - [ - ("returns", add_returns), - PipelineStep("volatility", add_volatility, dependencies=["returns"]), - ], - ) - - final_result = pipeline.run(sample_data) - - # Get intermediate result after returns step - returns_result = pipeline.get_intermediate_result("returns") - assert returns_result is not None - assert "returns" in returns_result.columns - assert "volatility" not in returns_result.columns - - # Get result after volatility step - volatility_result = pipeline.get_intermediate_result("volatility") - assert volatility_result is not None - assert "returns" in volatility_result.columns - assert "volatility" in volatility_result.columns - - # Check that final result matches the last intermediate result - assert final_result.equals(volatility_result) - - def test_step_with_parameters(self, sample_data: pl.DataFrame) -> None: - """Test pipeline steps with parameters.""" - - def add_rolling_mean( - df: pl.DataFrame, - window: int = 5, - column: str = "price", - ) -> pl.DataFrame: - return df.with_columns( - **{ - f"rolling_mean_{window}": pl.col(column).rolling_mean( - window_size=window, - ), - }, - ) - - pipeline = Pipeline( - [ - PipelineStep( - "rolling_mean_3", - add_rolling_mean, - params={"window": 3, "column": "price"}, - ), - PipelineStep( - "rolling_mean_5", - add_rolling_mean, - params={"window": 5, "column": "price"}, - ), - ], - ) - - result = pipeline.run(sample_data) - - assert "rolling_mean_3" in result.columns - assert "rolling_mean_5" in result.columns - - def test_deterministic_ordering(self, sample_data: pl.DataFrame) -> None: - """Test that pipeline execution is deterministic when multiple orderings are valid.""" - execution_logs = [] - - def make_step(name: str): - def step(df: pl.DataFrame) -> pl.DataFrame: - execution_logs[-1].append(name) - return df.with_columns(**{f"col_{name.lower()}": pl.lit(1)}) - - return step - - # Create pipeline where B and C can both execute after A - # but the algorithm should produce consistent ordering - for _ in range(5): # Run multiple times - execution_logs.append([]) - - pipeline = Pipeline( - [ - PipelineStep("C", make_step("C"), dependencies=["A"]), - PipelineStep("A", make_step("A")), - PipelineStep("B", make_step("B"), dependencies=["A"]), - ], - ) - - pipeline.run(sample_data) - - # All runs should have the same execution order (deterministic) - first_order = execution_logs[0] - for log in execution_logs[1:]: - assert log == first_order, f"Non-deterministic ordering: {log} != {first_order}" diff --git a/tests/test_risk.py b/tests/test_risk.py index 8f61479..7ac839f 100644 --- a/tests/test_risk.py +++ b/tests/test_risk.py @@ -394,6 +394,7 @@ def test_edge_cases(self): dd = result["dd"].drop_nulls() assert dd.max() < 1e-10 # Should be near zero + @pytest.mark.perf def test_performance(self): """Test performance with larger dataset.""" import time diff --git a/tests/test_roc_variants.py b/tests/test_roc_variants.py index 6944f02..566f455 100644 --- a/tests/test_roc_variants.py +++ b/tests/test_roc_variants.py @@ -241,6 +241,7 @@ def test_roc_variants_crypto_accuracy(self, crypto_data_small): rtol=1e-10, ) + @pytest.mark.perf @pytest.mark.benchmark def test_roc_variants_performance( self, diff --git a/tests/test_stochf.py b/tests/test_stochf.py index 08dca32..5afe5da 100644 --- a/tests/test_stochf.py +++ b/tests/test_stochf.py @@ -246,6 +246,7 @@ def test_stochf_comparison_with_stoch(self, price_data): # STOCHF %K should match assert_indicator_match(stochf_k, stoch_k, "STOCHF vs STOCH", rtol=1e-10) + @pytest.mark.perf @pytest.mark.benchmark def test_stochf_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark STOCHF performance.""" diff --git a/tests/test_t3.py b/tests/test_t3.py index a6335d9..4b5d083 100644 --- a/tests/test_t3.py +++ b/tests/test_t3.py @@ -186,6 +186,7 @@ def test_t3_crypto_accuracy(self, crypto_data_small): err_msg=f"T3 mismatch on crypto data for period {period}, vfactor {vfactor}", ) + @pytest.mark.perf @pytest.mark.benchmark def test_t3_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark T3 performance.""" diff --git a/tests/test_talib_accuracy.py b/tests/test_talib_accuracy.py index 2de9b0c..96c51f7 100644 --- a/tests/test_talib_accuracy.py +++ b/tests/test_talib_accuracy.py @@ -289,6 +289,7 @@ def test_different_data_types(self): assert_allclose(result_array, expected, rtol=1e-6, atol=1e-6) assert_allclose(result_series, expected, rtol=1e-6, atol=1e-6) + @pytest.mark.perf def test_performance_consistency(self): """Test that different implementations give same results.""" np.random.seed(42) @@ -689,6 +690,7 @@ def generate_data(self, n): close = 100 * (1 + returns).cumprod() return close + @pytest.mark.perf def test_sma_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark SMA implementations.""" # Use real crypto data @@ -727,6 +729,7 @@ def test_sma_performance(self, crypto_data, performance_threshold, warmup_jit): f"Performance ratio {our_time / talib_time:.1f}x exceeds threshold {threshold}x" ) + @pytest.mark.perf def test_rsi_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark RSI implementation.""" # Use real crypto data diff --git a/tests/test_talib_p0_indicators.py b/tests/test_talib_p0_indicators.py index 41b761c..1494dd8 100644 --- a/tests/test_talib_p0_indicators.py +++ b/tests/test_talib_p0_indicators.py @@ -259,7 +259,7 @@ def test_mom_accuracy(self, price_data): expected = talib.MOM(close, timeperiod=period) # Our implementation - result = mom(close, timeperiod=period) + result = mom(close, period=period) # Check accuracy assert_allclose( @@ -274,7 +274,7 @@ def test_mom_edge_cases(self, edge_cases): """Test MOM with edge cases.""" for name, data in edge_cases.items(): expected = talib.MOM(data, timeperiod=10) - result = mom(data, timeperiod=10) + result = mom(data, period=10) assert_allclose( result, @@ -400,6 +400,7 @@ def large_dataset(self): "volume": volume, } + @pytest.mark.perf @pytest.mark.benchmark def test_wma_performance(self, large_dataset, benchmark): """Benchmark WMA performance.""" @@ -416,6 +417,7 @@ def test_wma_performance(self, large_dataset, benchmark): expected = talib.WMA(close, timeperiod=period) assert_allclose(result, expected, rtol=1e-6, atol=1e-6) + @pytest.mark.perf @pytest.mark.benchmark def test_obv_performance(self, large_dataset, benchmark): """Benchmark OBV performance.""" @@ -432,6 +434,7 @@ def test_obv_performance(self, large_dataset, benchmark): expected = talib.OBV(close, volume) assert_allclose(result, expected, rtol=1e-6, atol=1e-6) + @pytest.mark.perf @pytest.mark.benchmark def test_mom_performance(self, large_dataset, benchmark): """Benchmark MOM performance.""" @@ -439,10 +442,10 @@ def test_mom_performance(self, large_dataset, benchmark): period = 10 # Warm up JIT - _ = mom(close[:1000], timeperiod=period) + _ = mom(close[:1000], period=period) # Benchmark our implementation - result = benchmark(mom, close, timeperiod=period) + result = benchmark(mom, close, period=period) # Verify correctness expected = talib.MOM(close, timeperiod=period) diff --git a/tests/test_timezone_handling.py b/tests/test_timezone_handling.py index 4f9aaaa..e44a264 100644 --- a/tests/test_timezone_handling.py +++ b/tests/test_timezone_handling.py @@ -60,14 +60,14 @@ def test_next_basic_open_naive(self): """Test next market open with naive datetime.""" cal = EquityCalendar() - # Monday 3 PM ET -> Tuesday 9:30 AM ET + # Monday 3 PM ET (during market hours) -> Tuesday 9:30 AM ET dt = datetime(2024, 1, 8, 15, 0, 0) next_open = cal._next_basic_open(dt) assert next_open.hour == 9 assert next_open.minute == 30 - assert next_open.day == 8 # Same day since before close + assert next_open.day == 9 # Next day (currently in session) - # Monday 5 PM ET -> Tuesday 9:30 AM ET + # Monday 5 PM ET (after close) -> Tuesday 9:30 AM ET dt = datetime(2024, 1, 8, 17, 0, 0) next_open = cal._next_basic_open(dt) assert next_open.day == 9 # Next day @@ -97,12 +97,12 @@ def test_previous_basic_close_naive(self): """Test previous market close with naive datetime.""" cal = EquityCalendar() - # Tuesday 10 AM ET -> Monday 4 PM ET + # Tuesday 10 AM ET (during market hours) -> Monday 4 PM ET dt = datetime(2024, 1, 9, 10, 0, 0) prev_close = cal._previous_basic_close(dt) assert prev_close.hour == 16 assert prev_close.minute == 0 - assert prev_close.day == 9 # Same day since after open + assert prev_close.day == 8 # Previous day (currently in session) # Tuesday 8 AM ET -> Monday 4 PM ET dt = datetime(2024, 1, 9, 8, 0, 0) diff --git a/tests/test_trange.py b/tests/test_trange.py index 91090da..8ddeaba 100644 --- a/tests/test_trange.py +++ b/tests/test_trange.py @@ -157,6 +157,7 @@ def test_trange_crypto_accuracy(self, crypto_data_small): # Should match exactly on real data assert_allclose(result, expected, rtol=1e-10, equal_nan=True) + @pytest.mark.perf @pytest.mark.benchmark def test_trange_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark TRANGE performance using real crypto data.""" diff --git a/tests/test_trix.py b/tests/test_trix.py index 6bf0055..8b8e915 100644 --- a/tests/test_trix.py +++ b/tests/test_trix.py @@ -145,6 +145,7 @@ def test_trix_crypto_accuracy(self, crypto_data_small): err_msg=f"TRIX mismatch on crypto data for period {period}", ) + @pytest.mark.perf @pytest.mark.benchmark def test_trix_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark TRIX performance using real crypto data.""" diff --git a/tests/test_tsf.py b/tests/test_tsf.py index 3f95902..5a9a379 100644 --- a/tests/test_tsf.py +++ b/tests/test_tsf.py @@ -165,6 +165,7 @@ def test_tsf_forecasting_property(self): valid_idx = ~np.isnan(result) assert len(result[valid_idx]) > 0 # Should have some valid forecasts + @pytest.mark.perf @pytest.mark.benchmark def test_tsf_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark TSF performance using real crypto data.""" diff --git a/tests/test_typprice.py b/tests/test_typprice.py index 1523fd1..b07234e 100644 --- a/tests/test_typprice.py +++ b/tests/test_typprice.py @@ -155,6 +155,7 @@ def test_typprice_crypto_accuracy(self, crypto_data_small): # Should match exactly on real data assert_allclose(result, expected, rtol=1e-10, equal_nan=True) + @pytest.mark.perf @pytest.mark.benchmark def test_typprice_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark TYPPRICE performance using real crypto data.""" diff --git a/tests/test_ultosc.py b/tests/test_ultosc.py index ebe96f1..1db0ccf 100644 --- a/tests/test_ultosc.py +++ b/tests/test_ultosc.py @@ -182,6 +182,7 @@ def test_ultosc_crypto_accuracy(self, crypto_data_small): err_msg=f"ULTOSC mismatch on crypto data for periods ({period1}, {period2}, {period3})", ) + @pytest.mark.perf @pytest.mark.benchmark def test_ultosc_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark ULTOSC performance using real crypto data.""" diff --git a/tests/test_var.py b/tests/test_var.py index e8439c0..e385f70 100644 --- a/tests/test_var.py +++ b/tests/test_var.py @@ -194,6 +194,7 @@ def test_var_crypto_accuracy(self, crypto_data_small): err_msg=f"VAR mismatch on crypto data for period {period}", ) + @pytest.mark.perf @pytest.mark.benchmark def test_var_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark VAR performance.""" diff --git a/tests/test_wclprice.py b/tests/test_wclprice.py index c583581..b017a38 100644 --- a/tests/test_wclprice.py +++ b/tests/test_wclprice.py @@ -171,6 +171,7 @@ def test_wclprice_crypto_accuracy(self, crypto_data_small): # Should match exactly on real data assert_allclose(result, expected, rtol=1e-10, equal_nan=True) + @pytest.mark.perf @pytest.mark.benchmark def test_wclprice_performance(self, crypto_data, performance_threshold, warmup_jit): """Benchmark WCLPRICE performance using real crypto data.""" diff --git a/tests/visualization/__init__.py b/tests/visualization/__init__.py deleted file mode 100644 index 5bbc0e0..0000000 --- a/tests/visualization/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests for visualization module.""" diff --git a/tests/visualization/test_summary.py b/tests/visualization/test_summary.py deleted file mode 100644 index 0803129..0000000 --- a/tests/visualization/test_summary.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Tests for plot export utility.""" - -from __future__ import annotations - -import pytest - -from ml4t.engineer.visualization import export_plot - - -class TestExportPlot: - """Test export_plot utility function.""" - - def test_export_requires_figure(self, tmp_path): - """Test that export_plot validates figure argument.""" - output_path = tmp_path / "test.png" - with pytest.raises((AttributeError, TypeError)): - export_plot(None, output_path) - - def test_export_invalid_extension(self, tmp_path): - """Test that export_plot validates file extension.""" - try: - import matplotlib.pyplot as plt - except ImportError: - pytest.skip("matplotlib not installed") - - fig, ax = plt.subplots() - ax.plot([1, 2, 3]) - - output_path = tmp_path / "test.xyz" - with pytest.raises(ValueError, match="Unsupported format"): - export_plot(fig, output_path) - - plt.close(fig) - - def test_export_creates_directory(self, tmp_path): - """Test that export_plot creates parent directories.""" - try: - import matplotlib.pyplot as plt - except ImportError: - pytest.skip("matplotlib not installed") - - fig, ax = plt.subplots() - ax.plot([1, 2, 3]) - - output_path = tmp_path / "subdir" / "test.png" - export_plot(fig, output_path, dpi=72) - - assert output_path.exists() - assert output_path.stat().st_size > 0 - - plt.close(fig) diff --git a/uv.lock b/uv.lock index 686d27e..ae1b1ef 100644 --- a/uv.lock +++ b/uv.lock @@ -1137,69 +1137,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9c/96/30f3fe51b336bb6da4714f4fdad7bbdce8f13af79af2eb75e22908f3f9f4/korean_lunar_calendar-0.3.1-py3-none-any.whl", hash = "sha256:392757135c492c4f42a604e6038042953c35c6f449dda5f27e3f86a7f9c943e5", size = 9033, upload-time = "2022-09-16T10:53:23.771Z" }, ] -[[package]] -name = "librt" -version = "0.7.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b7/29/47f29026ca17f35cf299290292d5f8331f5077364974b7675a353179afa2/librt-0.7.7.tar.gz", hash = "sha256:81d957b069fed1890953c3b9c3895c7689960f233eea9a1d9607f71ce7f00b2c", size = 145910, upload-time = "2026-01-01T23:52:22.87Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/56/30b5c342518005546df78841cb0820ae85a17e7d07d521c10ef367306d0d/librt-0.7.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a487b71fbf8a9edb72a8c7a456dda0184642d99cd007bc819c0b7ab93676a8ee", size = 54709, upload-time = "2026-01-01T23:51:02.774Z" }, - { url = "https://files.pythonhosted.org/packages/72/78/9f120e3920b22504d4f3835e28b55acc2cc47c9586d2e1b6ba04c3c1bf01/librt-0.7.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f4d4efb218264ecf0f8516196c9e2d1a0679d9fb3bb15df1155a35220062eba8", size = 56663, upload-time = "2026-01-01T23:51:03.838Z" }, - { url = "https://files.pythonhosted.org/packages/1c/ea/7d7a1ee7dfc1151836028eba25629afcf45b56bbc721293e41aa2e9b8934/librt-0.7.7-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b8bb331aad734b059c4b450cd0a225652f16889e286b2345af5e2c3c625c3d85", size = 161705, upload-time = "2026-01-01T23:51:04.917Z" }, - { url = "https://files.pythonhosted.org/packages/45/a5/952bc840ac8917fbcefd6bc5f51ad02b89721729814f3e2bfcc1337a76d6/librt-0.7.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:467dbd7443bda08338fc8ad701ed38cef48194017554f4c798b0a237904b3f99", size = 171029, upload-time = "2026-01-01T23:51:06.09Z" }, - { url = "https://files.pythonhosted.org/packages/fa/bf/c017ff7da82dc9192cf40d5e802a48a25d00e7639b6465cfdcee5893a22c/librt-0.7.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50d1d1ee813d2d1a3baf2873634ba506b263032418d16287c92ec1cc9c1a00cb", size = 184704, upload-time = "2026-01-01T23:51:07.549Z" }, - { url = "https://files.pythonhosted.org/packages/77/ec/72f3dd39d2cdfd6402ab10836dc9cbf854d145226062a185b419c4f1624a/librt-0.7.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7e5070cf3ec92d98f57574da0224f8c73faf1ddd6d8afa0b8c9f6e86997bc74", size = 180719, upload-time = "2026-01-01T23:51:09.062Z" }, - { url = "https://files.pythonhosted.org/packages/78/86/06e7a1a81b246f3313bf515dd9613a1c81583e6fd7843a9f4d625c4e926d/librt-0.7.7-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:bdb9f3d865b2dafe7f9ad7f30ef563c80d0ddd2fdc8cc9b8e4f242f475e34d75", size = 174537, upload-time = "2026-01-01T23:51:10.611Z" }, - { url = "https://files.pythonhosted.org/packages/83/08/f9fb2edc9c7a76e95b2924ce81d545673f5b034e8c5dd92159d1c7dae0c6/librt-0.7.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8185c8497d45164e256376f9da5aed2bb26ff636c798c9dabe313b90e9f25b28", size = 195238, upload-time = "2026-01-01T23:51:11.762Z" }, - { url = "https://files.pythonhosted.org/packages/ba/56/ea2d2489d3ea1f47b301120e03a099e22de7b32c93df9a211e6ff4f9bf38/librt-0.7.7-cp311-cp311-win32.whl", hash = "sha256:44d63ce643f34a903f09ff7ca355aae019a3730c7afd6a3c037d569beeb5d151", size = 42939, upload-time = "2026-01-01T23:51:13.192Z" }, - { url = "https://files.pythonhosted.org/packages/58/7b/c288f417e42ba2a037f1c0753219e277b33090ed4f72f292fb6fe175db4c/librt-0.7.7-cp311-cp311-win_amd64.whl", hash = "sha256:7d13cc340b3b82134f8038a2bfe7137093693dcad8ba5773da18f95ad6b77a8a", size = 49240, upload-time = "2026-01-01T23:51:14.264Z" }, - { url = "https://files.pythonhosted.org/packages/7c/24/738eb33a6c1516fdb2dfd2a35db6e5300f7616679b573585be0409bc6890/librt-0.7.7-cp311-cp311-win_arm64.whl", hash = "sha256:983de36b5a83fe9222f4f7dcd071f9b1ac6f3f17c0af0238dadfb8229588f890", size = 42613, upload-time = "2026-01-01T23:51:15.268Z" }, - { url = "https://files.pythonhosted.org/packages/56/72/1cd9d752070011641e8aee046c851912d5f196ecd726fffa7aed2070f3e0/librt-0.7.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2a85a1fc4ed11ea0eb0a632459ce004a2d14afc085a50ae3463cd3dfe1ce43fc", size = 55687, upload-time = "2026-01-01T23:51:16.291Z" }, - { url = "https://files.pythonhosted.org/packages/50/aa/d5a1d4221c4fe7e76ae1459d24d6037783cb83c7645164c07d7daf1576ec/librt-0.7.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c87654e29a35938baead1c4559858f346f4a2a7588574a14d784f300ffba0efd", size = 57136, upload-time = "2026-01-01T23:51:17.363Z" }, - { url = "https://files.pythonhosted.org/packages/23/6f/0c86b5cb5e7ef63208c8cc22534df10ecc5278efc0d47fb8815577f3ca2f/librt-0.7.7-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c9faaebb1c6212c20afd8043cd6ed9de0a47d77f91a6b5b48f4e46ed470703fe", size = 165320, upload-time = "2026-01-01T23:51:18.455Z" }, - { url = "https://files.pythonhosted.org/packages/16/37/df4652690c29f645ffe405b58285a4109e9fe855c5bb56e817e3e75840b3/librt-0.7.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1908c3e5a5ef86b23391448b47759298f87f997c3bd153a770828f58c2bb4630", size = 174216, upload-time = "2026-01-01T23:51:19.599Z" }, - { url = "https://files.pythonhosted.org/packages/9a/d6/d3afe071910a43133ec9c0f3e4ce99ee6df0d4e44e4bddf4b9e1c6ed41cc/librt-0.7.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dbc4900e95a98fc0729523be9d93a8fedebb026f32ed9ffc08acd82e3e181503", size = 189005, upload-time = "2026-01-01T23:51:21.052Z" }, - { url = "https://files.pythonhosted.org/packages/d5/18/74060a870fe2d9fd9f47824eba6717ce7ce03124a0d1e85498e0e7efc1b2/librt-0.7.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a7ea4e1fbd253e5c68ea0fe63d08577f9d288a73f17d82f652ebc61fa48d878d", size = 183961, upload-time = "2026-01-01T23:51:22.493Z" }, - { url = "https://files.pythonhosted.org/packages/7c/5e/918a86c66304af66a3c1d46d54df1b2d0b8894babc42a14fb6f25511497f/librt-0.7.7-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:ef7699b7a5a244b1119f85c5bbc13f152cd38240cbb2baa19b769433bae98e50", size = 177610, upload-time = "2026-01-01T23:51:23.874Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d7/b5e58dc2d570f162e99201b8c0151acf40a03a39c32ab824dd4febf12736/librt-0.7.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:955c62571de0b181d9e9e0a0303c8bc90d47670a5eff54cf71bf5da61d1899cf", size = 199272, upload-time = "2026-01-01T23:51:25.341Z" }, - { url = "https://files.pythonhosted.org/packages/18/87/8202c9bd0968bdddc188ec3811985f47f58ed161b3749299f2c0dd0f63fb/librt-0.7.7-cp312-cp312-win32.whl", hash = "sha256:1bcd79be209313b270b0e1a51c67ae1af28adad0e0c7e84c3ad4b5cb57aaa75b", size = 43189, upload-time = "2026-01-01T23:51:26.799Z" }, - { url = "https://files.pythonhosted.org/packages/61/8d/80244b267b585e7aa79ffdac19f66c4861effc3a24598e77909ecdd0850e/librt-0.7.7-cp312-cp312-win_amd64.whl", hash = "sha256:4353ee891a1834567e0302d4bd5e60f531912179578c36f3d0430f8c5e16b456", size = 49462, upload-time = "2026-01-01T23:51:27.813Z" }, - { url = "https://files.pythonhosted.org/packages/2d/1f/75db802d6a4992d95e8a889682601af9b49d5a13bbfa246d414eede1b56c/librt-0.7.7-cp312-cp312-win_arm64.whl", hash = "sha256:a76f1d679beccccdf8c1958e732a1dfcd6e749f8821ee59d7bec009ac308c029", size = 42828, upload-time = "2026-01-01T23:51:28.804Z" }, - { url = "https://files.pythonhosted.org/packages/8d/5e/d979ccb0a81407ec47c14ea68fb217ff4315521730033e1dd9faa4f3e2c1/librt-0.7.7-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f4a0b0a3c86ba9193a8e23bb18f100d647bf192390ae195d84dfa0a10fb6244", size = 55746, upload-time = "2026-01-01T23:51:29.828Z" }, - { url = "https://files.pythonhosted.org/packages/f5/2c/3b65861fb32f802c3783d6ac66fc5589564d07452a47a8cf9980d531cad3/librt-0.7.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5335890fea9f9e6c4fdf8683061b9ccdcbe47c6dc03ab8e9b68c10acf78be78d", size = 57174, upload-time = "2026-01-01T23:51:31.226Z" }, - { url = "https://files.pythonhosted.org/packages/50/df/030b50614b29e443607220097ebaf438531ea218c7a9a3e21ea862a919cd/librt-0.7.7-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9b4346b1225be26def3ccc6c965751c74868f0578cbcba293c8ae9168483d811", size = 165834, upload-time = "2026-01-01T23:51:32.278Z" }, - { url = "https://files.pythonhosted.org/packages/5d/e1/bd8d1eacacb24be26a47f157719553bbd1b3fe812c30dddf121c0436fd0b/librt-0.7.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a10b8eebdaca6e9fdbaf88b5aefc0e324b763a5f40b1266532590d5afb268a4c", size = 174819, upload-time = "2026-01-01T23:51:33.461Z" }, - { url = "https://files.pythonhosted.org/packages/46/7d/91d6c3372acf54a019c1ad8da4c9ecf4fc27d039708880bf95f48dbe426a/librt-0.7.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:067be973d90d9e319e6eb4ee2a9b9307f0ecd648b8a9002fa237289a4a07a9e7", size = 189607, upload-time = "2026-01-01T23:51:34.604Z" }, - { url = "https://files.pythonhosted.org/packages/fa/ac/44604d6d3886f791fbd1c6ae12d5a782a8f4aca927484731979f5e92c200/librt-0.7.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23d2299ed007812cccc1ecef018db7d922733382561230de1f3954db28433977", size = 184586, upload-time = "2026-01-01T23:51:35.845Z" }, - { url = "https://files.pythonhosted.org/packages/5c/26/d8a6e4c17117b7f9b83301319d9a9de862ae56b133efb4bad8b3aa0808c9/librt-0.7.7-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6b6f8ea465524aa4c7420c7cc4ca7d46fe00981de8debc67b1cc2e9957bb5b9d", size = 178251, upload-time = "2026-01-01T23:51:37.018Z" }, - { url = "https://files.pythonhosted.org/packages/99/ab/98d857e254376f8e2f668e807daccc1f445e4b4fc2f6f9c1cc08866b0227/librt-0.7.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8df32a99cc46eb0ee90afd9ada113ae2cafe7e8d673686cf03ec53e49635439", size = 199853, upload-time = "2026-01-01T23:51:38.195Z" }, - { url = "https://files.pythonhosted.org/packages/7c/55/4523210d6ae5134a5da959900be43ad8bab2e4206687b6620befddb5b5fd/librt-0.7.7-cp313-cp313-win32.whl", hash = "sha256:86f86b3b785487c7760247bcdac0b11aa8bf13245a13ed05206286135877564b", size = 43247, upload-time = "2026-01-01T23:51:39.629Z" }, - { url = "https://files.pythonhosted.org/packages/25/40/3ec0fed5e8e9297b1cf1a3836fb589d3de55f9930e3aba988d379e8ef67c/librt-0.7.7-cp313-cp313-win_amd64.whl", hash = "sha256:4862cb2c702b1f905c0503b72d9d4daf65a7fdf5a9e84560e563471e57a56949", size = 49419, upload-time = "2026-01-01T23:51:40.674Z" }, - { url = "https://files.pythonhosted.org/packages/1c/7a/aab5f0fb122822e2acbc776addf8b9abfb4944a9056c00c393e46e543177/librt-0.7.7-cp313-cp313-win_arm64.whl", hash = "sha256:0996c83b1cb43c00e8c87835a284f9057bc647abd42b5871e5f941d30010c832", size = 42828, upload-time = "2026-01-01T23:51:41.731Z" }, - { url = "https://files.pythonhosted.org/packages/69/9c/228a5c1224bd23809a635490a162e9cbdc68d99f0eeb4a696f07886b8206/librt-0.7.7-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:23daa1ab0512bafdd677eb1bfc9611d8ffbe2e328895671e64cb34166bc1b8c8", size = 55188, upload-time = "2026-01-01T23:51:43.14Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c2/0e7c6067e2b32a156308205e5728f4ed6478c501947e9142f525afbc6bd2/librt-0.7.7-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:558a9e5a6f3cc1e20b3168fb1dc802d0d8fa40731f6e9932dcc52bbcfbd37111", size = 56895, upload-time = "2026-01-01T23:51:44.534Z" }, - { url = "https://files.pythonhosted.org/packages/0e/77/de50ff70c80855eb79d1d74035ef06f664dd073fb7fb9d9fb4429651b8eb/librt-0.7.7-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2567cb48dc03e5b246927ab35cbb343376e24501260a9b5e30b8e255dca0d1d2", size = 163724, upload-time = "2026-01-01T23:51:45.571Z" }, - { url = "https://files.pythonhosted.org/packages/6e/19/f8e4bf537899bdef9e0bb9f0e4b18912c2d0f858ad02091b6019864c9a6d/librt-0.7.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6066c638cdf85ff92fc6f932d2d73c93a0e03492cdfa8778e6d58c489a3d7259", size = 172470, upload-time = "2026-01-01T23:51:46.823Z" }, - { url = "https://files.pythonhosted.org/packages/42/4c/dcc575b69d99076768e8dd6141d9aecd4234cba7f0e09217937f52edb6ed/librt-0.7.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a609849aca463074c17de9cda173c276eb8fee9e441053529e7b9e249dc8b8ee", size = 186806, upload-time = "2026-01-01T23:51:48.009Z" }, - { url = "https://files.pythonhosted.org/packages/fe/f8/4094a2b7816c88de81239a83ede6e87f1138477d7ee956c30f136009eb29/librt-0.7.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:add4e0a000858fe9bb39ed55f31085506a5c38363e6eb4a1e5943a10c2bfc3d1", size = 181809, upload-time = "2026-01-01T23:51:49.35Z" }, - { url = "https://files.pythonhosted.org/packages/1b/ac/821b7c0ab1b5a6cd9aee7ace8309c91545a2607185101827f79122219a7e/librt-0.7.7-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a3bfe73a32bd0bdb9a87d586b05a23c0a1729205d79df66dee65bb2e40d671ba", size = 175597, upload-time = "2026-01-01T23:51:50.636Z" }, - { url = "https://files.pythonhosted.org/packages/71/f9/27f6bfbcc764805864c04211c6ed636fe1d58f57a7b68d1f4ae5ed74e0e0/librt-0.7.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0ecce0544d3db91a40f8b57ae26928c02130a997b540f908cefd4d279d6c5848", size = 196506, upload-time = "2026-01-01T23:51:52.535Z" }, - { url = "https://files.pythonhosted.org/packages/46/ba/c9b9c6fc931dd7ea856c573174ccaf48714905b1a7499904db2552e3bbaf/librt-0.7.7-cp314-cp314-win32.whl", hash = "sha256:8f7a74cf3a80f0c3b0ec75b0c650b2f0a894a2cec57ef75f6f72c1e82cdac61d", size = 39747, upload-time = "2026-01-01T23:51:53.683Z" }, - { url = "https://files.pythonhosted.org/packages/c5/69/cd1269337c4cde3ee70176ee611ab0058aa42fc8ce5c9dce55f48facfcd8/librt-0.7.7-cp314-cp314-win_amd64.whl", hash = "sha256:3d1fe2e8df3268dd6734dba33ededae72ad5c3a859b9577bc00b715759c5aaab", size = 45971, upload-time = "2026-01-01T23:51:54.697Z" }, - { url = "https://files.pythonhosted.org/packages/79/fd/e0844794423f5583108c5991313c15e2b400995f44f6ec6871f8aaf8243c/librt-0.7.7-cp314-cp314-win_arm64.whl", hash = "sha256:2987cf827011907d3dfd109f1be0d61e173d68b1270107bb0e89f2fca7f2ed6b", size = 39075, upload-time = "2026-01-01T23:51:55.726Z" }, - { url = "https://files.pythonhosted.org/packages/42/02/211fd8f7c381e7b2a11d0fdfcd410f409e89967be2e705983f7c6342209a/librt-0.7.7-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8e92c8de62b40bfce91d5e12c6e8b15434da268979b1af1a6589463549d491e6", size = 57368, upload-time = "2026-01-01T23:51:56.706Z" }, - { url = "https://files.pythonhosted.org/packages/4c/b6/aca257affae73ece26041ae76032153266d110453173f67d7603058e708c/librt-0.7.7-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f683dcd49e2494a7535e30f779aa1ad6e3732a019d80abe1309ea91ccd3230e3", size = 59238, upload-time = "2026-01-01T23:51:58.066Z" }, - { url = "https://files.pythonhosted.org/packages/96/47/7383a507d8e0c11c78ca34c9d36eab9000db5989d446a2f05dc40e76c64f/librt-0.7.7-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9b15e5d17812d4d629ff576699954f74e2cc24a02a4fc401882dd94f81daba45", size = 183870, upload-time = "2026-01-01T23:51:59.204Z" }, - { url = "https://files.pythonhosted.org/packages/a4/b8/50f3d8eec8efdaf79443963624175c92cec0ba84827a66b7fcfa78598e51/librt-0.7.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c084841b879c4d9b9fa34e5d5263994f21aea7fd9c6add29194dbb41a6210536", size = 194608, upload-time = "2026-01-01T23:52:00.419Z" }, - { url = "https://files.pythonhosted.org/packages/23/d9/1b6520793aadb59d891e3b98ee057a75de7f737e4a8b4b37fdbecb10d60f/librt-0.7.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c8fb9966f84737115513fecbaf257f9553d067a7dd45a69c2c7e5339e6a8dc", size = 206776, upload-time = "2026-01-01T23:52:01.705Z" }, - { url = "https://files.pythonhosted.org/packages/ff/db/331edc3bba929d2756fa335bfcf736f36eff4efcb4f2600b545a35c2ae58/librt-0.7.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5fb1ecb2c35362eab2dbd354fd1efa5a8440d3e73a68be11921042a0edc0ff", size = 203206, upload-time = "2026-01-01T23:52:03.315Z" }, - { url = "https://files.pythonhosted.org/packages/b2/e1/6af79ec77204e85f6f2294fc171a30a91bb0e35d78493532ed680f5d98be/librt-0.7.7-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d1454899909d63cc9199a89fcc4f81bdd9004aef577d4ffc022e600c412d57f3", size = 196697, upload-time = "2026-01-01T23:52:04.857Z" }, - { url = "https://files.pythonhosted.org/packages/f3/46/de55ecce4b2796d6d243295c221082ca3a944dc2fb3a52dcc8660ce7727d/librt-0.7.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7ef28f2e7a016b29792fe0a2dd04dec75725b32a1264e390c366103f834a9c3a", size = 217193, upload-time = "2026-01-01T23:52:06.159Z" }, - { url = "https://files.pythonhosted.org/packages/41/61/33063e271949787a2f8dd33c5260357e3d512a114fc82ca7890b65a76e2d/librt-0.7.7-cp314-cp314t-win32.whl", hash = "sha256:5e419e0db70991b6ba037b70c1d5bbe92b20ddf82f31ad01d77a347ed9781398", size = 40277, upload-time = "2026-01-01T23:52:07.625Z" }, - { url = "https://files.pythonhosted.org/packages/06/21/1abd972349f83a696ea73159ac964e63e2d14086fdd9bc7ca878c25fced4/librt-0.7.7-cp314-cp314t-win_amd64.whl", hash = "sha256:d6b7d93657332c817b8d674ef6bf1ab7796b4f7ce05e420fd45bd258a72ac804", size = 46765, upload-time = "2026-01-01T23:52:08.647Z" }, - { url = "https://files.pythonhosted.org/packages/51/0e/b756c7708143a63fca65a51ca07990fa647db2cc8fcd65177b9e96680255/librt-0.7.7-cp314-cp314t-win_arm64.whl", hash = "sha256:142c2cd91794b79fd0ce113bd658993b7ede0fe93057668c2f98a45ca00b7e91", size = 39724, upload-time = "2026-01-01T23:52:09.745Z" }, -] - [[package]] name = "lightgbm" version = "4.6.0" @@ -1461,7 +1398,6 @@ all = [ { name = "ipython" }, { name = "lightgbm" }, { name = "matplotlib" }, - { name = "mypy" }, { name = "myst-parser" }, { name = "nbsphinx" }, { name = "pandas-market-calendars" }, @@ -1491,7 +1427,6 @@ dev = [ { name = "ipdb" }, { name = "ipython" }, { name = "lightgbm" }, - { name = "mypy" }, { name = "pandas-market-calendars" }, { name = "pre-commit" }, { name = "pytest" }, @@ -1532,6 +1467,7 @@ viz = [ [package.dev-dependencies] dev = [ { name = "hypothesis" }, + { name = "pandas-market-calendars" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-benchmark" }, @@ -1562,8 +1498,6 @@ requires-dist = [ { name = "matplotlib", specifier = ">=3.7.0" }, { name = "matplotlib", marker = "extra == 'all'", specifier = ">=3.7.0" }, { name = "matplotlib", marker = "extra == 'viz'", specifier = ">=3.7.0" }, - { name = "mypy", marker = "extra == 'all'", specifier = ">=1.5.0" }, - { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.5.0" }, { name = "myst-parser", marker = "extra == 'all'", specifier = ">=2.0.0" }, { name = "myst-parser", marker = "extra == 'docs'", specifier = ">=2.0.0" }, { name = "nbsphinx", marker = "extra == 'all'", specifier = ">=0.9.0" }, @@ -1620,6 +1554,7 @@ provides-extras = ["all", "calendars", "dev", "docs", "ml", "store", "ta", "viz" [package.metadata.requires-dev] dev = [ { name = "hypothesis", specifier = ">=6.80.0" }, + { name = "pandas-market-calendars", specifier = ">=4.0.0" }, { name = "pre-commit", specifier = ">=3.3.0" }, { name = "pytest", specifier = ">=7.4.0" }, { name = "pytest-benchmark", specifier = ">=4.0.0" }, @@ -1646,54 +1581,6 @@ version = "0.0.12" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/17/0d/74f0293dfd7dcc3837746d0138cbedd60b31701ecc75caec7d3f281feba0/multitasking-0.0.12.tar.gz", hash = "sha256:2fba2fa8ed8c4b85e227c5dd7dc41c7d658de3b6f247927316175a57349b84d1", size = 19984, upload-time = "2025-07-20T21:27:51.636Z" } -[[package]] -name = "mypy" -version = "1.19.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, - { name = "mypy-extensions" }, - { name = "pathspec" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f5/db/4efed9504bc01309ab9c2da7e352cc223569f05478012b5d9ece38fd44d2/mypy-1.19.1.tar.gz", hash = "sha256:19d88bb05303fe63f71dd2c6270daca27cb9401c4ca8255fe50d1d920e0eb9ba", size = 3582404, upload-time = "2025-12-15T05:03:48.42Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/47/6b3ebabd5474d9cdc170d1342fbf9dddc1b0ec13ec90bf9004ee6f391c31/mypy-1.19.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d8dfc6ab58ca7dda47d9237349157500468e404b17213d44fc1cb77bce532288", size = 13028539, upload-time = "2025-12-15T05:03:44.129Z" }, - { url = "https://files.pythonhosted.org/packages/5c/a6/ac7c7a88a3c9c54334f53a941b765e6ec6c4ebd65d3fe8cdcfbe0d0fd7db/mypy-1.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e3f276d8493c3c97930e354b2595a44a21348b320d859fb4a2b9f66da9ed27ab", size = 12083163, upload-time = "2025-12-15T05:03:37.679Z" }, - { url = "https://files.pythonhosted.org/packages/67/af/3afa9cf880aa4a2c803798ac24f1d11ef72a0c8079689fac5cfd815e2830/mypy-1.19.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2abb24cf3f17864770d18d673c85235ba52456b36a06b6afc1e07c1fdcd3d0e6", size = 12687629, upload-time = "2025-12-15T05:02:31.526Z" }, - { url = "https://files.pythonhosted.org/packages/2d/46/20f8a7114a56484ab268b0ab372461cb3a8f7deed31ea96b83a4e4cfcfca/mypy-1.19.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a009ffa5a621762d0c926a078c2d639104becab69e79538a494bcccb62cc0331", size = 13436933, upload-time = "2025-12-15T05:03:15.606Z" }, - { url = "https://files.pythonhosted.org/packages/5b/f8/33b291ea85050a21f15da910002460f1f445f8007adb29230f0adea279cb/mypy-1.19.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f7cee03c9a2e2ee26ec07479f38ea9c884e301d42c6d43a19d20fb014e3ba925", size = 13661754, upload-time = "2025-12-15T05:02:26.731Z" }, - { url = "https://files.pythonhosted.org/packages/fd/a3/47cbd4e85bec4335a9cd80cf67dbc02be21b5d4c9c23ad6b95d6c5196bac/mypy-1.19.1-cp311-cp311-win_amd64.whl", hash = "sha256:4b84a7a18f41e167f7995200a1d07a4a6810e89d29859df936f1c3923d263042", size = 10055772, upload-time = "2025-12-15T05:03:26.179Z" }, - { url = "https://files.pythonhosted.org/packages/06/8a/19bfae96f6615aa8a0604915512e0289b1fad33d5909bf7244f02935d33a/mypy-1.19.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a8174a03289288c1f6c46d55cef02379b478bfbc8e358e02047487cad44c6ca1", size = 13206053, upload-time = "2025-12-15T05:03:46.622Z" }, - { url = "https://files.pythonhosted.org/packages/a5/34/3e63879ab041602154ba2a9f99817bb0c85c4df19a23a1443c8986e4d565/mypy-1.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffcebe56eb09ff0c0885e750036a095e23793ba6c2e894e7e63f6d89ad51f22e", size = 12219134, upload-time = "2025-12-15T05:03:24.367Z" }, - { url = "https://files.pythonhosted.org/packages/89/cc/2db6f0e95366b630364e09845672dbee0cbf0bbe753a204b29a944967cd9/mypy-1.19.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b64d987153888790bcdb03a6473d321820597ab8dd9243b27a92153c4fa50fd2", size = 12731616, upload-time = "2025-12-15T05:02:44.725Z" }, - { url = "https://files.pythonhosted.org/packages/00/be/dd56c1fd4807bc1eba1cf18b2a850d0de7bacb55e158755eb79f77c41f8e/mypy-1.19.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c35d298c2c4bba75feb2195655dfea8124d855dfd7343bf8b8c055421eaf0cf8", size = 13620847, upload-time = "2025-12-15T05:03:39.633Z" }, - { url = "https://files.pythonhosted.org/packages/6d/42/332951aae42b79329f743bf1da088cd75d8d4d9acc18fbcbd84f26c1af4e/mypy-1.19.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34c81968774648ab5ac09c29a375fdede03ba253f8f8287847bd480782f73a6a", size = 13834976, upload-time = "2025-12-15T05:03:08.786Z" }, - { url = "https://files.pythonhosted.org/packages/6f/63/e7493e5f90e1e085c562bb06e2eb32cae27c5057b9653348d38b47daaecc/mypy-1.19.1-cp312-cp312-win_amd64.whl", hash = "sha256:b10e7c2cd7870ba4ad9b2d8a6102eb5ffc1f16ca35e3de6bfa390c1113029d13", size = 10118104, upload-time = "2025-12-15T05:03:10.834Z" }, - { url = "https://files.pythonhosted.org/packages/de/9f/a6abae693f7a0c697dbb435aac52e958dc8da44e92e08ba88d2e42326176/mypy-1.19.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e3157c7594ff2ef1634ee058aafc56a82db665c9438fd41b390f3bde1ab12250", size = 13201927, upload-time = "2025-12-15T05:02:29.138Z" }, - { url = "https://files.pythonhosted.org/packages/9a/a4/45c35ccf6e1c65afc23a069f50e2c66f46bd3798cbe0d680c12d12935caa/mypy-1.19.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdb12f69bcc02700c2b47e070238f42cb87f18c0bc1fc4cdb4fb2bc5fd7a3b8b", size = 12206730, upload-time = "2025-12-15T05:03:01.325Z" }, - { url = "https://files.pythonhosted.org/packages/05/bb/cdcf89678e26b187650512620eec8368fded4cfd99cfcb431e4cdfd19dec/mypy-1.19.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f859fb09d9583a985be9a493d5cfc5515b56b08f7447759a0c5deaf68d80506e", size = 12724581, upload-time = "2025-12-15T05:03:20.087Z" }, - { url = "https://files.pythonhosted.org/packages/d1/32/dd260d52babf67bad8e6770f8e1102021877ce0edea106e72df5626bb0ec/mypy-1.19.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9a6538e0415310aad77cb94004ca6482330fece18036b5f360b62c45814c4ef", size = 13616252, upload-time = "2025-12-15T05:02:49.036Z" }, - { url = "https://files.pythonhosted.org/packages/71/d0/5e60a9d2e3bd48432ae2b454b7ef2b62a960ab51292b1eda2a95edd78198/mypy-1.19.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:da4869fc5e7f62a88f3fe0b5c919d1d9f7ea3cef92d3689de2823fd27e40aa75", size = 13840848, upload-time = "2025-12-15T05:02:55.95Z" }, - { url = "https://files.pythonhosted.org/packages/98/76/d32051fa65ecf6cc8c6610956473abdc9b4c43301107476ac03559507843/mypy-1.19.1-cp313-cp313-win_amd64.whl", hash = "sha256:016f2246209095e8eda7538944daa1d60e1e8134d98983b9fc1e92c1fc0cb8dd", size = 10135510, upload-time = "2025-12-15T05:02:58.438Z" }, - { url = "https://files.pythonhosted.org/packages/de/eb/b83e75f4c820c4247a58580ef86fcd35165028f191e7e1ba57128c52782d/mypy-1.19.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06e6170bd5836770e8104c8fdd58e5e725cfeb309f0a6c681a811f557e97eac1", size = 13199744, upload-time = "2025-12-15T05:03:30.823Z" }, - { url = "https://files.pythonhosted.org/packages/94/28/52785ab7bfa165f87fcbb61547a93f98bb20e7f82f90f165a1f69bce7b3d/mypy-1.19.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:804bd67b8054a85447c8954215a906d6eff9cabeabe493fb6334b24f4bfff718", size = 12215815, upload-time = "2025-12-15T05:02:42.323Z" }, - { url = "https://files.pythonhosted.org/packages/0a/c6/bdd60774a0dbfb05122e3e925f2e9e846c009e479dcec4821dad881f5b52/mypy-1.19.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21761006a7f497cb0d4de3d8ef4ca70532256688b0523eee02baf9eec895e27b", size = 12740047, upload-time = "2025-12-15T05:03:33.168Z" }, - { url = "https://files.pythonhosted.org/packages/32/2a/66ba933fe6c76bd40d1fe916a83f04fed253152f451a877520b3c4a5e41e/mypy-1.19.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28902ee51f12e0f19e1e16fbe2f8f06b6637f482c459dd393efddd0ec7f82045", size = 13601998, upload-time = "2025-12-15T05:03:13.056Z" }, - { url = "https://files.pythonhosted.org/packages/e3/da/5055c63e377c5c2418760411fd6a63ee2b96cf95397259038756c042574f/mypy-1.19.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:481daf36a4c443332e2ae9c137dfee878fcea781a2e3f895d54bd3002a900957", size = 13807476, upload-time = "2025-12-15T05:03:17.977Z" }, - { url = "https://files.pythonhosted.org/packages/cd/09/4ebd873390a063176f06b0dbf1f7783dd87bd120eae7727fa4ae4179b685/mypy-1.19.1-cp314-cp314-win_amd64.whl", hash = "sha256:8bb5c6f6d043655e055be9b542aa5f3bdd30e4f3589163e85f93f3640060509f", size = 10281872, upload-time = "2025-12-15T05:03:05.549Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f4/4ce9a05ce5ded1de3ec1c1d96cf9f9504a04e54ce0ed55cfa38619a32b8d/mypy-1.19.1-py3-none-any.whl", hash = "sha256:f1235f5ea01b7db5468d53ece6aaddf1ad0b88d9e7462b86ef96fe04995d7247", size = 2471239, upload-time = "2025-12-15T05:03:07.248Z" }, -] - -[[package]] -name = "mypy-extensions" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, -] - [[package]] name = "myst-parser" version = "4.0.1" @@ -2005,15 +1892,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl", hash = "sha256:646204b5ee239c396d040b90f9e272e9a8017c630092bf59980beb62fd033887", size = 106668, upload-time = "2025-08-23T15:15:25.663Z" }, ] -[[package]] -name = "pathspec" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/28/2e/83722ece0f6ee24387d6cb830dd562ddbcd6ce0b9d76072c6849670c31b4/pathspec-1.0.1.tar.gz", hash = "sha256:e2769b508d0dd47b09af6ee2c75b2744a2cb1f474ae4b1494fd6a1b7a841613c", size = 129791, upload-time = "2026-01-06T13:02:55.15Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/fe/2257c71721aeab6a6e8aa1f00d01f2a20f58547d249a6c8fef5791f559fc/pathspec-1.0.1-py3-none-any.whl", hash = "sha256:8870061f22c58e6d83463cfce9a7dd6eca0512c772c1001fb09ac64091816721", size = 54584, upload-time = "2026-01-06T13:02:53.601Z" }, -] - [[package]] name = "patsy" version = "1.0.2" From 669c30aa528fd34bad38c7dad76330909ce82799 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 3 Mar 2026 12:38:21 -0500 Subject: [PATCH 2/3] fix(ci): build TA-Lib from source on Ubuntu Noble --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2f98a2..17825c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,8 +74,13 @@ jobs: - name: Install TA-Lib C library run: | - sudo apt-get update - sudo apt-get install -y libta-lib0-dev + wget -q http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz + tar -xzf ta-lib-0.4.0-src.tar.gz + cd ta-lib/ + ./configure --prefix=/usr/local + make -j$(nproc) + sudo make install + sudo ldconfig - name: Install dependencies run: uv sync --dev --extra ta From 3602b63d2b6216821c1dead832463def1883ff5f Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 3 Mar 2026 12:40:07 -0500 Subject: [PATCH 3/3] fix(ci): remove parallel make for TA-Lib build (race condition) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17825c6..1711525 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,7 @@ jobs: tar -xzf ta-lib-0.4.0-src.tar.gz cd ta-lib/ ./configure --prefix=/usr/local - make -j$(nproc) + make sudo make install sudo ldconfig