diff --git a/nextplace/validator/README.md b/nextplace/validator/README.md index ba5bce1..c6fa3fc 100644 --- a/nextplace/validator/README.md +++ b/nextplace/validator/README.md @@ -1,10 +1,11 @@ # Validator -## API Key -You need a [Redfin API](https://rapidapi.com/ntd119/api/redfin-com-data) key to run this validator. Select the ULTRA subscription at $35 per month. -You must store your API key in a `.env` file at the root of this repository, in a field called `NEXT_PLACE_REDFIN_API_KEY` +## API Keys +You need a [Redfin API](https://rapidapi.com/ntd119/api/redfin-com-data) key for the US and a [Redfine API](https://rapidapi.com/ntd119/api/redfin-canada) key for Canada to run this validator. Select the ULTRA subscription for both, at $35 per month. +You must store your API key in a `.env` file at the root of this repository, in a field called `NEXT_PLACE_REDFIN_API_KEY` for US markets, and `NEXTPLACE_CANADA_API_KEY` for Canada. This may be the same key for both, but it should still be stored in two separate variables. - EX `.env` - ```NEXT_PLACE_REDFIN_API_KEY="your-api-key"``` +- ```NEXTPLACE_CANADA_API_KEY="your-api-key"``` ## Setup Steps Note: Runpod and Vast are not recommended. Validating this subnet does not require a GPU. diff --git a/nextplace/validator/api/api_base.py b/nextplace/validator/api/api_base.py index 3726dfe..5953b8e 100644 --- a/nextplace/validator/api/api_base.py +++ b/nextplace/validator/api/api_base.py @@ -8,51 +8,78 @@ """ Abstract base class contains data global to all API calls """ - - class ApiBase(ABC): - def __init__(self, database_manager: DatabaseManager, markets: list[dict[str, str]]): self.nextplace_hash_key = b'next_place_hash_key_3b1f2aebc9d8e456' # For creating the nextplace_id self.database_manager = database_manager self.markets = markets - api_key = self._get_api_key_from_env() + + # Load API keys from environment + load_dotenv() + self.us_api_key = os.getenv("NEXT_PLACE_REDFIN_API_KEY") + self.canada_api_key = os.getenv("NEXTPLACE_CANADA_API_KEY") + + # Default US headers self.headers = { - "X-RapidAPI-Key": api_key, + "X-RapidAPI-Key": self.us_api_key, "X-RapidAPI-Host": "redfin-com-data.p.rapidapi.com" } + + # Canadian headers + self.canada_headers = { + "X-RapidAPI-Key": self.canada_api_key, + "X-RapidAPI-Host": "redfin-canada.p.rapidapi.com" + } + self.max_results_per_page = 350 # This is typically the maximum allowed by Redfin's API - + def get_hash(self, address: str, zip_code: str) -> str: """ Build the nextplace_id using a 1-way cryptographic hash function Args: address: the home's street address zip_code: the home's zip code - + Returns: the cryptographic hash of the address-zip """ message = f"{address}-{zip_code}" hashed = hmac.new(self.nextplace_hash_key, message.encode(), hashlib.sha256) return hashed.hexdigest() - - def _get_api_key_from_env(self) -> str: + + def get_headers(self, market_id: str) -> dict: """ - Load the API key from the environment + Get the appropriate headers based on market ID + Args: + market_id: The market identifier + Returns: - The redfin API key + The appropriate headers for the API request """ - load_dotenv() - return os.getenv("NEXT_PLACE_REDFIN_API_KEY") - + if market_id.startswith('33'): + return self.canada_headers + return self.headers + + def get_api_url(self, endpoint: str, market_id: str) -> str: + """ + Get the appropriate API URL based on market ID + Args: + endpoint: The API endpoint (e.g., 'search-sale', 'search-sold') + market_id: The market identifier + + Returns: + The complete API URL + """ + base_url = "https://redfin-canada.p.rapidapi.com/properties/" if market_id.startswith('33') else "https://redfin-com-data.p.rapidapi.com/properties/" + return f"{base_url}{endpoint}" + def _get_nested(self, data: dict, *args: str) -> dict or None: """ Extract nested values from a dictionary Args: data: the dictionary *args: - + Returns: A dictionary or None """ @@ -62,3 +89,4 @@ def _get_nested(self, data: dict, *args: str) -> dict or None: else: return None return data + \ No newline at end of file diff --git a/nextplace/validator/api/properties_api.py b/nextplace/validator/api/properties_api.py index 5f5513c..c30700c 100644 --- a/nextplace/validator/api/properties_api.py +++ b/nextplace/validator/api/properties_api.py @@ -29,7 +29,15 @@ def process_region_market(self, market: dict[str, str]) -> None: None """ current_thread = threading.current_thread().name - url_for_sale = "https://redfin-com-data.p.rapidapi.com/properties/search-sale" # Redfin URL + + # Choose the appropriate API endpoint based on market ID + if market['id'].startswith('33'): + url_for_sale = "https://redfin-canada.p.rapidapi.com/properties/search-sale" # Canadian Redfin URL + headers = self.canada_headers # Use Canadian API key + else: + url_for_sale = "https://redfin-com-data.p.rapidapi.com/properties/search-sale" # US Redfin URL + headers = self.headers # Use US API key + page = 1 # Page number for api results while True: @@ -40,7 +48,7 @@ def process_region_market(self, market: dict[str, str]) -> None: "limit": self.max_results_per_page, "page": page } - response = requests.get(url_for_sale, headers=self.headers, params=querystring) # Hit the API + response = requests.get(url_for_sale, headers=headers, params=querystring) # Only proceed with status code is 200 if response.status_code != 200: @@ -145,3 +153,4 @@ def _build_property_object(self, home_data: any) -> Home: 'last_sale_date': self._get_nested(home_data, 'lastSaleData', 'lastSoldDate'), 'hoa_dues': self._get_nested(home_data, 'hoaDues', 'amount'), } + \ No newline at end of file diff --git a/nextplace/validator/api/sold_homes_api.py b/nextplace/validator/api/sold_homes_api.py index 1c484ab..2329308 100644 --- a/nextplace/validator/api/sold_homes_api.py +++ b/nextplace/validator/api/sold_homes_api.py @@ -43,7 +43,15 @@ def _process_region_sold_homes(self, market: dict) -> None: """ current_thread = threading.current_thread().name region_id = market['id'] - url_sold = "https://redfin-com-data.p.rapidapi.com/properties/search-sold" # URL for sold houses + + # Choose the appropriate API endpoint and headers based on market ID + if region_id.startswith('33'): + url_sold = "https://redfin-canada.p.rapidapi.com/properties/search-sold" # Canadian URL for sold houses + headers = self.canada_headers # Use Canadian API key + else: + url_sold = "https://redfin-com-data.p.rapidapi.com/properties/search-sold" # US URL for sold houses + headers = self.headers # Use US API key + page = 1 # Page number for api results invalid_results = {'date': 0, 'price': 0, 'timezone': 0} @@ -59,7 +67,7 @@ def _process_region_sold_homes(self, market: dict) -> None: "page": page } - response = requests.get(url_sold, headers=self.headers, params=querystring) # Get API response + response = requests.get(url_sold, headers=headers, params=querystring) # Get API response # Only proceed with status code is 200 if response.status_code != 200: @@ -132,3 +140,4 @@ def _ingest_valid_homes(self, result_tuples: list[tuple]) -> None: VALUES (?, ?, ?, ?) """ self.database_manager.query_and_commit_many(query_str, result_tuples) + \ No newline at end of file diff --git a/nextplace/validator/market/markets.py b/nextplace/validator/market/markets.py index 226fcfe..b5f3f7c 100644 --- a/nextplace/validator/market/markets.py +++ b/nextplace/validator/market/markets.py @@ -630,5 +630,45 @@ { "name": "Sparks", "id": "6_17527" + }, + { + "name": "Toronto", + "id": "33_2924" + }, + { + "name": "Calgary", + "id": "33_2379" + }, + { + "name": "Edmonton", + "id": "33_3352" + }, + { + "name": "Mississauga", + "id": "33_2775" + }, + { + "name": "Brampton", + "id": "33_3446" + }, + { + "name": "Hamilton", + "id": "33_3114" + }, + { + "name": "Kitchener", + "id": "33_2996" + }, + { + "name": "London", + "id": "33_3505" + }, + { + "name": "Victoria", + "id": "33_2620" + }, + { + "name": "Markham", + "id": "33_2035" } ]