works. ready to tag and attempt a first release

pyproject.toml

@@ -1,8 +1,15 @@
 [project]
-name = "megalinref"
+name            = "megalinref"
+version         = "0.1.0"
+description     = "Find SLK and Road Number from lat/lon (and vice versa) using Main Roads Western Australia open data portal data."
+readme          = "readme.md"
+requires-python = ">=3.9"
 author = "Nicholas Archer"
 dependencies = ["pandas", "requests", "arcgis2geojson"]
 
+[project.urls]
+repository      = "https://github.com/thehappycheese/megalinref"
+
 [build-system]
 requires = ["maturin>=0.12,<0.13"]
 build-backend = "maturin"

python/megalinref/__init__.py

@@ -1 +1,6 @@
-from .data_handling import *
+from .megalinref import (
+    SLKLookup,
+    Cwy,
+    NetworkType
+)
+from ._data_handling import download_fresh_data_as_json

python/megalinref/_data_handling.py

@@ -0,0 +1,42 @@
+from typing import Any, Optional
+import requests
+from arcgis2geojson import arcgis2geojson
+import math
+
+DATA_SOURCE_URL = "https://mrgis.mainroads.wa.gov.au/arcgis/rest/services/OpenData/RoadAssets_DataPortal/MapServer/17/query?where=1%3D1&outFields=ROAD,START_SLK,END_SLK,CWY,NETWORK_TYPE,START_TRUE_DIST,END_TRUE_DIST&outSR=4326&f=json"
+
+
+def download_fresh_data_as_json(url:str=DATA_SOURCE_URL, chunk_limit:Optional[int]=None) -> dict[str, Any]:
+
+    response = requests.request("GET", f"{url}&returnCountOnly=true")
+    record_count = response.json()["count"]
+
+    print(f"Downloading {record_count} records" + (":" if chunk_limit is None else f", {chunk_limit=}:"))
+    if chunk_limit is not None:
+        print("." * min(chunk_limit, math.floor(record_count/1000)))
+    else:
+        print("." * math.floor(record_count/1000))
+
+    # TODO: make this download multiple chunks in parallel
+
+    output=[]
+    offset = 0
+    chunk_counter = 0
+    while True:
+        if chunk_limit is not None and chunk_counter >= chunk_limit:
+            break
+        chunk_counter += 1
+
+        response = requests.request("GET", f"{url}&resultOffset={offset}")
+        json = response.json()
+        if json["geometryType"] != "esriGeometryPolyline":
+            raise Exception("Rest service returned an object that did not have geometryType:esriGeometryPolyline")
+        offset += len(json["features"])
+        output.extend(json["features"])
+        if "exceededTransferLimit" not in json or not json["exceededTransferLimit"]:
+            break
+        print(".", end="")
+    print(f"\nDownload Completed. received {len(output)} records")
+    json["features"] = output
+    json = arcgis2geojson(json)
+    return json

src/lib.rs

@@ -14,35 +14,36 @@ use datatypes::{
 use slk_from_lat_lon::SLKLookup;
 
 
-
+/// My Module test documentation
 #[pymodule]
-fn megalinref(_py: Python, m: &PyModule) -> PyResult<()> {
-
-    let cwy_dict = PyDict::from_sequence(
-        _py, 
-        PyList::new(_py, &vec![
-            ("L", Cwy::L),
-            ("S", Cwy::S),
-            ("R", Cwy::R),
-        ]).to_object(_py)
-    ).unwrap();
-
-    let network_type_dict = PyDict::from_sequence(
-        _py, 
-        PyList::new(_py, &vec![
-            ("State Road",                 NetworkType::State_Road),
-            ("Local Road",                 NetworkType::Local_Road),
-            ("Miscellaneous Road",         NetworkType::Miscellaneous_Road),
-            ("Main Roads Controlled_Path", NetworkType::Main_Roads_Controlled_Path),
-            ("Proposed Road",              NetworkType::Proposed_Road),
-            ("Crossover",                  NetworkType::Crossover),
-        ]).to_object(_py)
-    ).unwrap();
+fn megalinref(py: Python, module: &PyModule) -> PyResult<()> {
+
+    // Cwy enum represented as dict
+    module.add("Cwy", PyDict::from_sequence(
+            py, 
+            PyList::new(py, &vec![
+                ("L", Cwy::L),
+                ("S", Cwy::S),
+                ("R", Cwy::R),
+            ]).to_object(py)
+        ).unwrap()
+    )?;
 
-
-    m.add("CWY", cwy_dict)?;
-    m.add("NETWORK_TYPE", network_type_dict)?;
-    m.add_class::<SLKLookup>()?;
+    // NetworkType enum represented as dict
+    module.add("NetworkType", PyDict::from_sequence(
+            py, 
+            PyList::new(py, &vec![
+                ("State Road",                 NetworkType::State_Road),
+                ("Local Road",                 NetworkType::Local_Road),
+                ("Miscellaneous Road",         NetworkType::Miscellaneous_Road),
+                ("Main Roads Controlled_Path", NetworkType::Main_Roads_Controlled_Path),
+                ("Proposed Road",              NetworkType::Proposed_Road),
+                ("Crossover",                  NetworkType::Crossover),
+            ]).to_object(py)
+        ).unwrap()
+    )?;
+
+    module.add_class::<SLKLookup>()?;
 
     Ok(())
 }