From cf0bdf7baf70fc49ca1beebb508681a177188ac9 Mon Sep 17 00:00:00 2001 From: Ludvig Ericson Date: Sat, 16 Aug 2014 16:02:44 +0200 Subject: [PATCH 1/3] Update README, add setup script --- README.md | 230 ++++++++++++++++++++++++++++++---------------------- booliapi.py | 1 - setup.py | 8 ++ 3 files changed, 142 insertions(+), 97 deletions(-) create mode 100644 setup.py diff --git a/README.md b/README.md index c6d467c..af76165 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,59 @@ +# Booli API + A simple and powerful python wrapper for the Booli.se API -# Setting up +## Setting up - from booliapi import BooliAPI - Booli = BooliAPI("your username", "your api key") +```python +from booliapi import BooliAPI +Booli = BooliAPI("your username", "your api key") +``` -# Searching +## Searching `Booli.search()` performs a search and returns a list of `Listing`s. Here's how you specify your search parameters: -## Searching by city and/or neighborhood +### Searching by city and/or neighborhood To search for listings in a given city or neighborhood, pass a string as the first argument to `.search()`. - Booli.search("Uppsala") # city +```python +Booli.search("Uppsala") # city +Booli.search("Uppsala/Luthagen") # city + neighborhood +``` + +#### Example: Printing a nice list of apartments in Uppsala's "Fålhagen" neighborhood. - Booli.search("Uppsala/Luthagen") # city + neighborhood +```python +for listing in Booli.search("Uppsala/Fålhagen", typ=u"lägenhet"): + print "%s, %s" % (listing.address, listing.neighborhood) + print " %s rum; %d kr, %d kr/mån" % (listing.rooms_as_text, + listing.price, + listing.fee) +``` -### Example: Printing a nice list of apartments in Uppsala's "Fålhagen" neighborhood. - - for listing in Booli.search("Uppsala/Fålhagen", typ=u"lägenhet"): - print "%s, %s" % (listing.address, listing.neighborhood) - print " %s rum; %d kr, %d kr/mån" % (listing.rooms_as_text, - listing.price, - listing.fee) - Result: - Hjalmar Brantingsgatan 9B, Fålhagen - 2 rum; 1490000 kr, 2710 kr/mån - Torkelsgatan 8C, Fålhagen - 1 rum; 1075000 kr, 2563 kr/mån - Petterslundsgatan 33, Fålhagen - 1 rum; 1050000 kr, 2032 kr/mån - ... +``` +Hjalmar Brantingsgatan 9B, Fålhagen + 2 rum; 1490000 kr, 2710 kr/mån +Torkelsgatan 8C, Fålhagen + 1 rum; 1075000 kr, 2563 kr/mån +Petterslundsgatan 33, Fålhagen + 1 rum; 1050000 kr, 2032 kr/mån +... +``` -## Listings within distance of a point +### Listings within distance of a point - # Listings within 1 km from Booli's Uppsala office - listings = Booli.search(centerLat=59.8569131, centerLong=17.6359056, radius=1) +```python +# Listings within 1 km from Booli's Uppsala office +listings = Booli.search(centerLat=59.8569131, centerLong=17.6359056, radius=1) +``` -## More specific searching +### More specific searching Any keyword arguments you pass to `.search()` will simply be used as URL parameters in the API query. Unicode strings, integers and floats are @@ -54,15 +65,16 @@ also valid keyword arguments here. [API Documentation]: http://www.booli.se/api/docs/ - Booli.search("Uppsala", pris="0-1500000", typ="lägenhet") - Booli.search("Stockholm/Södermalm", rum=[1,2]) - Booli.search("Stockholm", typ=["villa", "radhus"], rum=4) +```python +Booli.search("Uppsala", pris="0-1500000", typ="lägenhet") +Booli.search("Stockholm/Södermalm", rum=[1,2]) +Booli.search("Stockholm", typ=["villa", "radhus"], rum=4) +``` It also means that the names of these parameters are all Swedish. But then again, so are you, probably. Puss på dig. - -# Listings +## Listings The objects we're dealing with are called `Listing`s. The data about each listing is stored as attributes. @@ -74,13 +86,13 @@ each listing is stored as attributes. - `neighborhood` and `city` - exactly what you think - `rooms` - number of rooms - `size` - square meterage -- `lot_size` - size of the lot +- `lot\_size` - size of the lot - `price` - the listed sales price - `fee` - monthly fee *Note:* The number of rooms is actually a float, since some listings are specified as having for example 2.5 rooms. There is a special - property called `rooms_as_text` that gives you a nicer string + property called `rooms\_as\_text` that gives you a nicer string representation. **Additional geographic data** @@ -92,13 +104,13 @@ each listing is stored as attributes. **Metadata** - `url` - the url to this listing on booli.se -- `image_url` - the url to a thumbnail image, if available +- `image\_url` - the url to a thumbnail image, if available - `agency` - the name of the real estate agency representing the seller - `created` - when this listing was created - `id` - internal Booli ID -# Filtering, sorting and grouping +## Filtering, sorting and grouping It's super-easy to filter and sort the listings. @@ -106,7 +118,7 @@ The list you get from `.search()` is actually a clever subclass of `list`, called `ResultSet`. If you're familiar with Django's QuerySet API, you'll like this. -## Filtering +### Filtering `ResultSet.filter(**kwargs)` and `ResultSet.exclude(**kwargs)` @@ -122,24 +134,28 @@ If you specify multiple parameters, they are combined using boolean `and`. One way to use these methods is to do more specific filtering than the API itself supports. - # Get only listings from a specific agency - listings = Booli.search("Uppsala").filter(agency=u"Widerlöv & Co") +```python +# Get only listings from a specific agency +listings = Booli.search("Uppsala").filter(agency=u"Widerlöv & Co") +``` Another way is to work on different subsets of a search result without having to make another API call. - listings = Booli.search("Uppsala", typ=u"lägenhet") +```python +listings = Booli.search("Uppsala", typ=u"lägenhet") - for listing in listings.filter(neighborhood=u"Fålhagen"): - # do something +for listing in listings.filter(neighborhood=u"Fålhagen"): + # do something - for listing in listings.filter(neighborhood=u"Luthagen"): - # do something else +for listing in listings.filter(neighborhood=u"Luthagen"): + # do something else +```python Filtering ResultSets never affects the underlying API calls; it only creates a filtered copy the results you've already fetched. -## Filter operators +### Filter operators Just as in Django's QuerySets, you can do more than just exact matching. When you type `.filter(attr=value)`, it's actually @@ -157,51 +173,59 @@ Here are the operators and their plain-python equivalents: - `attr__contains=value` - `value in attr` - `attr__startswith=value` - `attr.startswith(value)` - `attr__endswith=value`- `attr.endswith(value)` -- `attr__range=(start, end)` - `start <= attr <= end` +- `attr__range=(start, end)` - `start <= attr <= end` There's also `iexact`, `icontains`, `istartswith` and `iendswith`, which are case-insensitive variants of their i-less buddies. -### Example: Finding all apartments on a specific street in Uppsala +#### Example: Finding all apartments on a specific street in Uppsala - apts = Booli.search("Uppsala", typ=u"lägenhet") +```python +apts = Booli.search("Uppsala", typ=u"lägenhet") +apts.filter(address__startswith="Storgatan") +``` - apts.filter(address__startswith="Storgatan") +#### Example: Getting listings in any of several neighborhoods -### Example: Getting listings in any of several neighborhoods +```python +apts.filter(neighborhood__in=[u"Luthagen", u"Centrum"]) +``` - apts.filter(neighborhood__in=[u"Luthagen", u"Centrum"]) +#### Example: Getting listings from one agency, excluding a neighborhood -### Example: Getting listings from one agency, excluding a neighborhood +```python +apts.filter(agency=u"Riksmäklaren").exclude(neighborhood=u"Sävja") +``` - apts.filter(agency=u"Riksmäklaren").exclude(neighborhood=u"Sävja") +#### Example: Getting listings published in the last 8 hours -### Example: Getting listings published in the last 8 hours +```python +from datetime import datetime, timedelta +eight_hours_ago = datetime.now() - timedelta(hours=8) - from datetime import datetime, timedelta - eight_hours_ago = datetime.now() - timedelta(hours=8) +for listing in apts.filter(created__gt=eight_hours_ago) + # do something +``` - for listing in apts.filter(created__gt=eight_hours_ago) - # do something - -## Sorting +### Sorting `ResultSet.order_by(*attributes)` `order_by()` takes one or more strings that specify which attributes to sort by. It returns a new ResultSet. - # Sort by address - apts.order_by("address") - - # Sort by price, descending (most expensive first) - apts.order_by("-price") +```python +# Sort by address +apts.order_by("address") - # Sort by neighborhood first, then by price descending - apts.order_by("neighborhood", "-price") +# Sort by price, descending (most expensive first) +apts.order_by("-price") +# Sort by neighborhood first, then by price descending +apts.order_by("neighborhood", "-price") +``` -## Grouping +### Grouping `ResultSet.group_by(attribute, [count_only=False])` @@ -215,28 +239,32 @@ group's resultset instead of the actual resultset. In almost all cases, you should sort your resultset before grouping. -### Example: Top 5 Agencies in Södermalm, Stockholm +#### Example: Top 5 Agencies in Södermalm, Stockholm - results = Booli.search("Stockholm/Södermalm").order_by("broker").group_by("broker") - results.sort(key=lambda x: len(x[1]), reverse=True) - for broker, listings in results[:5]: - print "%s (%d listings)" % (broker, len(listings)) - print - other = sum(len(listings) for (broker, listings) in results[5:]) - print u"Other: %d listings" % (other,) +```python +results = Booli.search("Stockholm/Södermalm").order_by("broker").group_by("broker") +results.sort(key=lambda x: len(x[1]), reverse=True) +for broker, listings in results[:5]: + print "%s (%d listings)" % (broker, len(listings)) +print +other = sum(len(listings) for (broker, listings) in results[5:]) +print u"Other: %d listings" % (other,) +``` Result: - Fastighetsbyrån (29 listings) - Svensk Fastighetsförmedling (25 listings) - Erik Olsson Fastighetsförmedling (17 listings) - Södermäklarna (13 listings) - Notar (12 listings) - - Other: 90 listings +``` +Fastighetsbyrån (29 listings) +Svensk Fastighetsförmedling (25 listings) +Erik Olsson Fastighetsförmedling (17 listings) +Södermäklarna (13 listings) +Notar (12 listings) + +Other: 90 listings +``` -## Complex filtering - Q and F objects +### Complex filtering - Q and F objects When you provide more than one parameter to `filter` or `exclude` they are combined using boolean AND. If that's not good enough for you, @@ -249,39 +277,49 @@ They can be combined using the `&` and `|` operators, and negated using `~`. These operations yield new Q objects representing the combined filter condition. -### Example: Finding apartments on any of several streets +#### Example: Finding apartments on any of several streets - kungsgatan = Q(address__startswith="Kungsgatan") - storgatan = Q(address__startswith="Storgatan") - Booli.search("Uppsala").filter(kungsgatan | storgatan) +```python +kungsgatan = Q(address__startswith="Kungsgatan") +storgatan = Q(address__startswith="Storgatan") +Booli.search("Uppsala").filter(kungsgatan | storgatan) +``` `F(attribute)` If you want to use a listing attribute as the right-hand side in a comparison, you have to use `F` objects. -### Example: Exploring the difference between City and Municipality in the data +#### Example: Exploring the difference between City and Municipality in the data - Booli.search("Uppsala").exclude(city=F("municipality")) +```python +Booli.search("Uppsala").exclude(city=F("municipality")) +``` `F` objects are very similar to `operator.attrgetter`, but with the addition that they can be combined with other `F` objects as well as with constants. -### Example: For rural living, try listings where the address is neighborhood + " " +#### Example: For rural living, try listings where the address is neighborhood + " " (+ number, but we can't search for that) - Booli.search("Uppsala").filter(address__startswith=F("neighborhood") + " ") +```python +Booli.search("Uppsala").filter(address__startswith=F("neighborhood") + " ") +``` -### Example: Grand living - finding a house where the lot is at least fifty times the size of the house +#### Example: Grand living - finding a house where the lot is at least fifty times the size of the house (avoiding those that oddly have size=0) - Booli.search("Uppsala", typ="villa").exclude(size=0) \ - .filter(lot_size__gte=F("size")*50) +```python +Booli.search("Uppsala", typ="villa").exclude(size=0) \ + .filter(lot_size__gte=F("size")*50) +``` -### Example: Filtering apartments that cost less than 25000/sqm +#### Example: Filtering apartments that cost less than 25000/sqm - Booli.search("Uppsala", typ=u"lägenhet").filter(price__lt=F("size")*25000) +```python +Booli.search("Uppsala", typ=u"lägenhet").filter(price__lt=F("size")*25000) +``` diff --git a/booliapi.py b/booliapi.py index d93610f..1a33e59 100644 --- a/booliapi.py +++ b/booliapi.py @@ -5,7 +5,6 @@ import json import operator import random -import re import string import urllib2 from datetime import datetime diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3928656 --- /dev/null +++ b/setup.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +from distutils.core import setup + +setup(name="booliapi", version="1.0.0", + url="http://github.com/byfilip/booliapi", + author="Filip Salomonsson", author_email="filip.salomonsson@gmail.com", + py_modules=['booliapi']) From ffb2d26685d4806769cfdbd49bb26e68238e7dfc Mon Sep 17 00:00:00 2001 From: Ludvig Ericson Date: Sat, 16 Aug 2014 16:18:50 +0200 Subject: [PATCH 2/3] Add credential storage --- README.md | 8 ++++++++ booliapi.py | 29 +++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index af76165..89b1fae 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,14 @@ from booliapi import BooliAPI Booli = BooliAPI("your username", "your api key") ``` +For convenience, these can also be stored in `~/.boolirc` as JSON: + +```json +{"caller_id": "your username", "key": "your api key"} +``` + +Then simply omit the arguments, so `Booli = BooliAPI()`. + ## Searching `Booli.search()` performs a search and returns a list of `Listing`s. diff --git a/booliapi.py b/booliapi.py index 1a33e59..30b7b3d 100644 --- a/booliapi.py +++ b/booliapi.py @@ -1,15 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import itertools +import os import json -import operator import random import string import urllib2 -from datetime import datetime +import operator +import itertools from hashlib import sha1 from urllib import urlencode +from datetime import datetime __version__ = "0.0" @@ -154,9 +155,29 @@ def group_by(self, key, count_only=False): class BooliAPI(object): base_url = "http://api.booli.se/listing/" - def __init__(self, caller_id, key): + def __init__(self, caller_id=None, key=None): + if not caller_id or not key: + d = self._load_user() + caller_id = caller_id or d.get('caller_id') + key = key or d.get('key') + + if not caller_id or not key: + raise ValueError("caller_id or key not given, and no " + "default found in ~/.boolirc") + self.caller_id = caller_id self.key = key + + def _load_user(self): + try: + f = open(os.path.expanduser("~/.boolirc"), "rb") + except IOError: + return {} + + try: + return json.load(f) + finally: + f.close() def search(self, area="", **params): url = self._build_url(area, params) From 65d26e1a18df5d15ba6d8739e8b3325b83b05864 Mon Sep 17 00:00:00 2001 From: Ludvig Ericson Date: Sat, 16 Aug 2014 16:35:15 +0200 Subject: [PATCH 3/3] Fix API discrepancies --- booliapi.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/booliapi.py b/booliapi.py index 30b7b3d..3bc0293 100644 --- a/booliapi.py +++ b/booliapi.py @@ -3,6 +3,7 @@ import os import json +import time import random import string import urllib2 @@ -35,7 +36,7 @@ "size": _float, "lot_size": _float, "rooms": _float, "lat": _float, "lon": _float, "id": _int, "fee": _int, "price": _int, - "created": lambda x: datetime.strptime(x, "%Y-%m-%d %H:%M:%S"), + "published": lambda x: datetime.strptime(x, "%Y-%m-%d %H:%M:%S"), } filterops = { @@ -153,7 +154,7 @@ def group_by(self, key, count_only=False): for key, group in itertools.groupby(self, key=F(key))] class BooliAPI(object): - base_url = "http://api.booli.se/listing/" + base_url = "http://api.booli.se/listings" def __init__(self, caller_id=None, key=None): if not caller_id or not key: @@ -178,26 +179,25 @@ def _load_user(self): return json.load(f) finally: f.close() - + def search(self, area="", **params): url = self._build_url(area, params) - response = urllib2.urlopen(url) - data = json.load(response) - content = data["booli"]["content"] + req = urllib2.Request(url, headers={"Accept": "application/vnd.booli-v2+json"}) + response = urllib2.urlopen(req) + content = json.load(response) resultset = ResultSet([Listing(item) for item in content["listings"]]) - resultset.total_count = content["totalListingCount"] + resultset.total_count = content["totalCount"] return resultset def _build_url(self, area, params): """Return a complete API request URL for the given search parameters, including the required authentication bits.""" - time = datetime.now().replace(microsecond=0).isoformat() + t = str(int(time.time())) unique = "".join(random.choice(string.letters + string.digits) for _ in range(16)) - hash = sha1(self.caller_id + time + self.key + unique).hexdigest() - params.update(callerId=self.caller_id, time=time, unique=unique, - hash=hash, format="json") - return self.base_url + area + "?" + smart_urlencode(params) + hash = sha1(self.caller_id + t + self.key + unique).hexdigest() + params.update(q=area, callerId=self.caller_id, time=t, unique=unique, hash=hash) + return self.base_url + "?" + smart_urlencode(params) class Listing(object):