Json importer

README.rst

@@ -1,3 +1,6 @@
+This is a fork of the original brightway2-io. My intention is to be able to import USLCI database using the JSON-LD importer. 
+===========================
+
 Brightway2 input and output
 ===========================
 

bw2io/extractors/__init__.py

@@ -6,3 +6,4 @@
 from .exiobase import Exiobase3MonetaryDataExtractor
 from .simapro_csv import SimaProCSVExtractor
 from .simapro_lcia_csv import SimaProLCIACSVExtractor
+from .json_ld import JSONLDExtractor

bw2io/importers/__init__.py

@@ -13,3 +13,5 @@
 from .exiobase3_monetary import Exiobase3MonetaryImporter
 from .simapro_csv import SimaProCSVImporter
 from .simapro_lcia_csv import SimaProLCIACSVImporter
+from .json_ld import JSONLDImporter
+from .json_ld_lcia import JSONLDLCIAImporter

bw2io/importers/json_ld.py

@@ -132,7 +132,7 @@ def boolcheck(lst):
             {
                 "code": obj["@id"],
                 "name": obj["name"],
-                "categories": category_mapping[obj["category"]["@id"]],
+                "categories": category_mapping[obj["category"]["@id"]], #There is no 'category'... does he mean 'type'?
                 "CAS number": obj.get("cas"),
                 "database": database_name + suffix,
                 "exchanges": [],

bw2io/strategies/json_ld.py

@@ -1,7 +1,7 @@
 from ..units import normalize_units as normalize_units_function
 
 
-def json_ld_get_normalized_exchange_locations(data):
+def json_ld_get_normalized_exchange_locations(data): #Makes a dictionary of the locations in the database
     """The exchanges location strings are not necessarily the same as those given in the process or the master metadata. Fix this inconsistency.
 
     This has to happen before we transform the input data from a dictionary to a list of activities, as it uses the ``locations`` data."""
@@ -53,6 +53,7 @@ def json_ld_convert_unit_to_reference_unit(db):
     for ds in db["processes"].values():
         for exc in ds["exchanges"]:
             unit_obj = exc.pop("unit")
+            #unit_obj = exc['flow'].pop('refUnit')
             exc["amount"] *= unit_conversion[unit_obj["@id"]]
             if "refUnit" in exc["flow"]:
                 exc["unit"] = exc["flow"].pop("refUnit")
@@ -78,7 +79,8 @@ def json_ld_add_activity_unit(db):
         production_exchanges = [
             exc
             for exc in ds["exchanges"]
-            if exc["flow"]["flowType"] == "PRODUCT_FLOW" and not exc["input"]
+            # if exc["flow"]["flowType"] == "PRODUCT_FLOW" and not exc["input"] and 'quantitativeReference' in exc.keys()
+            if (exc["flow"]["flowType"] == "PRODUCT_FLOW") & (~exc["input"]) & (exc.get('quantitativeReference') is True)
         ]
         assert len(production_exchanges) == 1, "Failed allocation"
         ds["unit"] = production_exchanges[0]["unit"]
@@ -119,6 +121,14 @@ def json_ld_remove_fields(db):
         "@context",
         "processType",
         "infrastructureProcess",
+        "processDocumentation",
+        "lastInternalId",
+        "description",
+        "version",
+        "dqSystem",
+        "dqEntry",
+        "exchangeDqSystem"
+
     }
 
     for ds in db:
@@ -130,9 +140,11 @@ def json_ld_remove_fields(db):
 
 def json_ld_location_name(db):
     for ds in db:
-        if ds.get("type") in {"emission", "product"}:
-            continue
-        ds["location"] = ds["location"]["name"]
+        if ds.get("type") not in {"emission", "product"}:
+            if 'location' not in ds.keys():
+               print(ds) #Fix this!
+            else:
+                ds["location"] = ds["location"]["name"]
 
     return db
 
@@ -174,18 +186,24 @@ def json_ld_label_exchange_type(db):
                 exc["type"] = "biosphere"
             elif exc.get("avoidedProduct"):
                 if exc.get("input"):
-                    raise ValueError("Avoided products are outputs, not inputs")
+                    # print('Activity name:', act['name'], act['code'], 'Exchange:', exc['flow']['name'])
+                     raise ValueError("Avoided products are outputs, not inputs", exc['flow']['name'])
                 exc["type"] = "substitution"
             elif exc["input"]:
-                if not exc.get("flow", {}).get("flowType") == "PRODUCT_FLOW":
+                if not exc.get("flow", {}).get("flowType") in ("PRODUCT_FLOW", "WASTE_FLOW"):   # I am going to allow waste flows to be technosphere inputs
                     raise ValueError("Inputs must be products")
                 exc["type"] = "technosphere"
             else:
                 if not exc.get("flow", {}).get("flowType") in (
                     "PRODUCT_FLOW",
                     "WASTE_FLOW",
                 ):
+                # This one looks like is jsut calling any PRODUCT FLOW or WASTE FLOW
+                # a 'production' type, production is only if the output is a reference flow.
                     raise ValueError("Outputs must be products")
+                # if exc.get("flow", {}).get("flowType") == "PRODUCT_FLOW":
+                #     exc["type"] = "production"
+                # elif exc.get("flow", {}).get("flowType") == "WASTE_FLOW":
+                #     exc["type"] = "waste"
                 exc["type"] = "production"
-
     return db

bw2io/strategies/json_ld_allocation.py

@@ -106,6 +106,9 @@ def json_ld_allocate_datasets(db, preferred_allocation=None):
 
     new_datasets = {}
 
+    if 'processes' not in db.keys():
+        print(db)
+
     for ds in db["processes"].values():
         if not allocation_needed(ds):
             continue

bw2io/strategies/json_ld_lcia.py

@@ -3,7 +3,8 @@ def json_ld_lcia_add_method_metadata(data):
         for category in value["impactCategories"]:
             obj = data["lcia_categories"][category["@id"]]
             obj["parent"] = {
-                k: value[k] for k in ("name", "description", "version", "lastChange")
+                # k: value[k] for k in ("name", "description", "version", "lastChange")
+                k: value[k] for k in ("name", "description", "version")
             }
     return data
 
@@ -17,7 +18,8 @@ def json_ld_lcia_set_method_metadata(data):
         if "referenceUnitName" in method:
             method["unit"] = method.pop("referenceUnitName")
         else:
-            method["unit"] = ""
+            continue
+        #method["unit"] = " " #This one gives trouble for some reason
         if "id" not in method:
             method["id"] = method.pop("@id")
         if not isinstance(method['name'], tuple):

bw2io/strategies/special.py

@@ -3,8 +3,10 @@ def add_dummy_processes_and_rename_exchanges(db):
     new_processes = set()
     for ds in db:
         for exc in ds.get("exchanges"):
-            if exc["name"][:6].lower() in ("dummy_", "dummy,"):
-                name = exc["name"][6:].lower().strip()
+            # if exc["name"][:6].lower() in ("dummy_", "dummy,"):
+            if "input" not in exc or "amount" not in exc:
+                # name = exc["name"][6:].lower().strip()
+                name = exc["name"].lower().strip()
                 new_processes.add(name)
                 exc["input"] = (ds["database"], name)
 
@@ -14,7 +16,7 @@ def add_dummy_processes_and_rename_exchanges(db):
                 "name": name,
                 "database": ds["database"],
                 "code": name,
-                "categories": ("dummy",),
+                "classifications": {name:"dummy",},
                 "location": "GLO",
                 "type": "process",
                 "exchanges": [