Add e-Stat provider for Japan government statistical data

openbb_platform/core/openbb_core/provider/standard_models/estat_search.py

@@ -0,0 +1,53 @@
+"""e-Stat Search Model."""
+
+from datetime import date as dateType
+from typing import Optional
+
+from openbb_core.provider.abstract.data import Data
+from openbb_core.provider.abstract.query_params import QueryParams
+from pydantic import Field
+
+
+class SearchQueryParams(QueryParams):
+    """e-Stat Search Query Params."""
+
+    query: Optional[str] = Field(
+        default=None,
+        description="Search keyword(s) for datasets. If not provided, returns all available datasets.",
+    )
+
+
+class SearchData(Data):
+    """e-Stat Search Data."""
+
+    dataset_id: str = Field(
+        description="Statistical dataset ID (e.g., '0003433219'). Use this with estat_series() to fetch data."
+    )
+    title: Optional[str] = Field(
+        default=None,
+        description="Dataset title in English (when available) or Japanese.",
+    )
+    stats_code: Optional[str] = Field(
+        default=None,
+        description="Statistical survey code (e.g., '00200521').",
+    )
+    gov_org: Optional[str] = Field(
+        default=None,
+        description="Government organization responsible for the dataset.",
+    )
+    statistics_name: Optional[str] = Field(
+        default=None,
+        description="Name of the statistical survey or collection.",
+    )
+    survey_date: Optional[dateType] = Field(
+        default=None,
+        description="Survey or data collection date.",
+    )
+    open_date: Optional[dateType] = Field(
+        default=None,
+        description="Date the dataset was made publicly available.",
+    )
+    small_area: Optional[str] = Field(
+        default=None,
+        description="Whether dataset includes small area statistics (0=No, 1=Yes).",
+    )

openbb_platform/dev_install.py

@@ -25,6 +25,7 @@
 openbb-benzinga = { path = "./providers/benzinga", develop = true }
 openbb-bls = { path = "./providers/bls", develop = true }
 openbb-cftc = { path = "./providers/cftc", develop = true }
+openbb-estat = { path = "./providers/estat", develop = true }
 openbb-congress-gov = { path = "./providers/congress_gov", develop = true }
 openbb-econdb = { path = "./providers/econdb", develop = true }
 openbb-federal-reserve = { path = "./providers/federal_reserve", develop = true }

openbb_platform/extensions/economy/integration/test_economy_python.py

@@ -338,6 +338,31 @@ def test_economy_fred_search(params, obb):
     assert len(result.results) > 0
 
 
+@pytest.mark.parametrize(
+    "params",
+    [
+        ({"provider": "estat"}),
+        ({"query": "population", "provider": "estat"}),
+        ({"query": "consumer price", "limit": 10, "provider": "estat"}),
+    ],
+)
+@pytest.mark.integration
+def test_economy_estat_search(params, obb):
+    """Test economy estat search."""
+    params = {p: v for p, v in params.items() if v}
+
+    result = obb.economy.estat_search(**params)
+    assert result
+    assert isinstance(result, OBBject)
+    assert len(result.results) > 0
+    # Verify dataset_id is present
+    assert result.results[0].dataset_id
+    # Verify metadata contains attribution
+    if hasattr(result, "extra") and result.extra:
+        metadata = result.extra.get("results_metadata", {})
+        assert "attribution" in metadata or len(metadata) > 0
+
+
 @pytest.mark.parametrize(
     "params",
     [

openbb_platform/extensions/economy/openbb_economy/economy_router.py

@@ -400,6 +400,77 @@ async def indicators(
     return await OBBject.from_query(Query(**locals()))
 
 
+@router.command(
+    model="EstatSearch",
+    examples=[
+        APIEx(
+            parameters={"provider": "estat"},
+            description="Get a list of all available e-Stat datasets.",
+        ),
+        APIEx(
+            parameters={"query": "population census", "provider": "estat"},
+            description="Search for datasets related to population census.",
+        ),
+        APIEx(
+            parameters={"query": "consumer price", "limit": 10, "provider": "estat"},
+            description="Search for consumer price datasets, limited to 10 results.",
+        ),
+    ],
+)
+async def estat_search(
+    cc: CommandContext,
+    provider_choices: ProviderChoices,
+    standard_params: StandardParams,
+    extra_params: ExtraParams,
+) -> OBBject:
+    """Search e-Stat datasets by keyword.
+
+    Returns dataset metadata including IDs, titles, and descriptions.
+    Use the dataset_id from results to query data with estat_series().
+
+    Metadata including attribution notice is available in response.extra['results_metadata'].
+
+    Attribution: This service uses API functions from e-Stat,
+    however its contents are not guaranteed by government.
+    """
+    return await OBBject.from_query(Query(**locals()))
+
+
+@router.command(
+    model="EstatStatisticalData",
+    examples=[
+        APIEx(
+            parameters={"symbol": "0003433219", "provider": "estat"},
+            description="Get Japanese population statistics data.",
+        ),
+        APIEx(
+            parameters={
+                "symbol": "0003433219",
+                "area_code": "00000",
+                "provider": "estat",
+            },
+            description="Get data for a specific area code.",
+        ),
+    ],
+)
+async def estat_series(
+    cc: CommandContext,
+    provider_choices: ProviderChoices,
+    standard_params: StandardParams,
+    extra_params: ExtraParams,
+) -> OBBject:
+    """Get Japanese government statistical data from e-Stat.
+
+    e-Stat is the portal site for Japanese government statistics.
+    It provides access to economic, demographic, and social statistics
+    from various ministries and agencies.
+
+    Attribution: This service uses API functions from e-Stat,
+    however its contents are not guaranteed by government.
+    """
+    return await OBBject.from_query(Query(**locals()))
+
+
 @router.command(
     model="CentralBankHoldings",
     examples=[

openbb_platform/providers/estat/README.md

@@ -0,0 +1,223 @@
+# OpenBB e-Stat Provider
+
+## Overview
+
+The e-Stat provider integrates Japan's official government statistical portal API into OpenBB. [e-Stat](https://www.e-stat.go.jp/en) is operated by the Ministry of Internal Affairs and Communications and provides access to 28+ statistical databases including population census, labor surveys, economic indicators, trade statistics, and regional data.
+
+**Attribution Notice**: This service uses API functions from e-Stat, however its contents are not guaranteed by government.
+
+## Setup Instructions
+
+### Step 1: Register for e-Stat Account
+
+1. Visit: https://www.e-stat.go.jp/en/mypage/user/preregister
+2. Complete registration with your email
+3. Confirm your email and log in
+
+### Step 2: Get Your Application ID (API Key)
+
+1. After logging in, go to "My Page" (top right corner)
+2. Click on "API functions" section
+3. Fill in the simple form:
+   - **Name**: Your app name
+   - **URL**: Your URL (use `http://test.localhost/` if not public)
+   - **Overview**: Brief description (optional)
+4. Click "Issue" button
+5. Your Application ID will be displayed immediately
+
+![e-Stat Application ID Dashboard](./appId.png)
+*Example of the API functions dashboard showing Application IDs*
+
+### Step 3: Configure Your API Key
+
+```python
+from openbb import obb
+
+# Set your API key (required)
+obb.account.credentials.estat_api_key = "your_application_id_here"
+```
+
+## Quick Start
+
+```python
+from openbb import obb
+
+# Search for datasets
+results = obb.economy.estat_search(query="population", provider="estat")
+dataset_id = results.results[0].dataset_id
+
+# Get Japanese population census data using the dataset ID
+data = obb.economy.estat_series(
+    provider="estat",
+    symbol=dataset_id,
+)
+
+# Or use a known dataset ID directly
+data = obb.economy.estat_series(
+    provider="estat",
+    symbol="0003433219",  # Population Census dataset ID
+    area_code="13000",    # Tokyo Prefecture
+    start_date="2020-01",
+    end_date="2020-12"
+)
+```
+
+## Finding Your Data
+
+### Understanding the Data Structure
+
+Think of e-Stat as Japan's official government data library. To use it effectively, understand these key concepts:
+
+**Dataset IDs** (like `0003433219`): Unique identifiers for each data collection. You'll find these on the e-Stat website when browsing datasets.
+
+**Area Codes**: Geographic filters
+- `00000` = All of Japan (national data)
+- `13000` = Tokyo Prefecture  
+- `27000` = Osaka Prefecture
+- City codes are longer (e.g., `13101` for Chiyoda-ku, Tokyo)
+
+**Stats Codes**: Survey identifiers (e.g., `00200521` for Population Census)
+
+**Category Codes**: Vary by dataset - check the specific dataset documentation for valid values
+
+### Three Ways to Find Dataset IDs
+
+#### Method 1: Browse the Website (Easiest)
+
+1. Go to: https://www.e-stat.go.jp/en/stat-search/database
+2. Use categories like "Population and Households", "Labor and Wages", "Prices"
+3. Click on a dataset to view details
+4. The dataset ID appears in the URL and dataset information
+
+#### Method 2: Search by Keywords
+
+Use the search box on the e-Stat website with English terms like "population", "unemployment", "consumer price"
+
+#### Method 3: Programmatic Search (Recommended)
+
+Use the built-in search function to discover datasets directly within OpenBB:
+
+```python
+from openbb import obb
+
+# Search for datasets by keyword
+results = obb.economy.estat_search(query="population census", provider="estat")
+
+# View search results
+for dataset in results.results:
+    print(f"ID: {dataset.dataset_id}")
+    print(f"Title: {dataset.title}")
+    print(f"Stats Code: {dataset.stats_code}")
+    print(f"Organization: {dataset.gov_org}")
+    print("---")
+
+# Access metadata and attribution
+print(results.extra['results_metadata']['attribution'])
+
+# Use a dataset ID from search results to fetch data
+dataset_id = results.results[0].dataset_id
+data = obb.economy.estat_series(symbol=dataset_id, provider="estat")
+```
+
+**Search Options:**
+- Omit `query` to get all available datasets
+- Use `limit` parameter to control number of results (default: 100, e-Stat API default: 10,000)
+- Use `start_position` for pagination (starts at 1). Get next page position from `response.extra['results_metadata']['next_start_position']`
+- Use `updated_date` to filter by dataset update date (format: YYYY-MM-DD)
+
+**Pagination Example:**
+```python
+# Get first 100 results
+page1 = obb.economy.estat_search(query="population", limit=100, provider="estat")
+
+# Check if more results available
+if page1.extra['results_metadata'].get('has_more'):
+    next_position = page1.extra['results_metadata']['next_start_position']
+    # Get next 100 results
+    page2 = obb.economy.estat_search(
+        query="population",
+        limit=100,
+        start_position=next_position,
+        provider="estat"
+    )
+```
+
+### Common Datasets
+
+**Population Census**: `0003433219` and similar IDs
+- Complete population count every 5 years
+- Includes demographics by prefecture/city
+
+**Consumer Price Index**: Various IDs in the `0003` series
+- Monthly inflation data
+- Price changes by category
+
+**Labor Force Survey**: Multiple IDs in the `0003` series
+- Monthly employment statistics
+- Unemployment rates
+
+Note: Dataset IDs can change when surveys are updated. Always verify current IDs on the e-Stat website.
+
+## Parameters
+
+- `symbol`: Statistical dataset ID (required) - the unique identifier from e-Stat
+- `stats_code`: Survey code like `00200521` for specific surveys
+- `area_code`: Geographic filter (see area codes above)
+- `category_code`: Dataset-specific category filters
+- `search_kind`: "1" for standard statistics, "2" for regional mesh statistics
+- `collect_area`: Collection area for aggregated data
+- `start_date` / `end_date`: Date range in YYYY-MM format
+
+**Note**: This provider currently only supports JSON format responses.
+
+## Troubleshooting
+
+**"Invalid Application ID" Error**
+- Verify your API key from the e-Stat API functions page
+- Ensure you've set `obb.account.credentials.estat_api_key`
+
+**"Statistical data not found" Error**
+- Check the dataset ID is current (IDs can change)
+- Not all parameters work with all datasets
+- Try removing optional parameters
+
+**Japanese Text in Responses**
+- Normal for some metadata despite `lang=E` parameter
+- Numerical data is universal
+
+**Empty Data**
+- Some datasets update infrequently (annually or every 5 years)
+- Check the e-Stat website for update schedules
+
+## API Reference
+
+- **API Guide**: https://www.e-stat.go.jp/api/api/index.php/en/api-info/api-guide
+- **API Specification**: https://www.e-stat.go.jp/api/api/index.php/en/api-info/api-spec
+- **Database Search**: https://www.e-stat.go.jp/en/stat-search/database
+
+## Development
+
+### Running Tests
+
+```bash
+cd openbb_platform/providers/estat
+poetry install
+poetry run pytest tests/
+```
+
+### Contributing
+
+Contributions are welcome! Areas for improvement:
+- Additional data fetchers for specific statistical categories
+- Enhanced date parsing for Japanese fiscal periods
+- Caching for frequently accessed metadata
+
+Please ensure all contributions maintain the attribution notice and comply with e-Stat's terms of use.
+
+## License & Attribution
+
+When using this provider, you must acknowledge:
+
+> This service uses API functions from e-Stat, however its contents are not guaranteed by government.
+
+This attribution is required by e-Stat's terms of use: https://www.e-stat.go.jp/api/api/index.php/en/api-info/credit