Fix import from bw2 generated packages #220
Conversation
|
Thanks @tfardet! I think we need some tests here. I can do this, but if you already have some older (and small :) fixtures then you are welcome to add them. I will try this weekend, but am not promising anything, there is already a list. |
|
OK, I tried to add the very simple attached package (which I generated with brightway 2) If I do from bw2io import BW2Package
BW2Package.import_file("bw2_compat_test.bw2package")it works fine, however, the following test import os
import pytest
from bw2data.tests import bw2test
from bw2io import BW2Package
FIXTURES = os.path.join(os.path.dirname(__file__), "..", "fixtures", "bw2package")
@bw2test
def test_bw2_compat():
obj = BW2Package.import_file(os.path.join(FIXTURES, "bw2_compat_test.bw2package"))[0]
a = obj.get("7599062216496486961")
self.assertTrue(a["name"] == "partial_respiration")
self.assertTrue(a["unit"] == "g")
self.assertTrue(a["type"] == "process")
a = obj.get("3866902554231372371")
self.assertTrue(a["name"] == "C_inactivated")
self.assertTrue(a["unit"] == "g")
self.assertTrue(a["type"] == "production")fails on import @bw2test
def test_bw2_compat():
> obj = BW2Package.import_file(os.path.join(FIXTURES, "bw2_compat_test.bw2package"))[0]
tests/bw2package/compat_bw2_import.py:13:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
bw2io/package.py:238: in import_file
return [cls._create_obj(o) for o in loaded]
bw2io/package.py:238: in <listcomp>
return [cls._create_obj(o) for o in loaded]
bw2io/package.py:129: in _create_obj
instance.write(data["data"])
../../../.pipenv/lib/python3.11/site-packages/bw2data/project.py:432: in writable_project
return wrapped(*args, **kwargs)
../../../.pipenv/lib/python3.11/site-packages/bw2data/backends/base.py:535: in write
self.process()
../../../.pipenv/lib/python3.11/site-packages/bw2data/backends/base.py:755: in process
dp.add_persistent_vector_from_iterator(
../../../.pipenv/lib/python3.11/site-packages/bw_processing/datapackage.py:441: in add_persistent_vector_from_iterator
) = resolve_dict_iterator(dict_iterator, nrows)
../../../.pipenv/lib/python3.11/site-packages/bw_processing/utils.py:73: in resolve_dict_iterator
array = create_structured_array(
../../../.pipenv/lib/python3.11/site-packages/bw_processing/array_creation.py:82: in create_structured_array
array = create_chunked_structured_array(iterable, dtype)
../../../.pipenv/lib/python3.11/site-packages/bw_processing/array_creation.py:44: in create_chunked_structured_array
for chunk in chunked(iterable, bucket_size):
../../../.pipenv/lib/python3.11/site-packages/bw_processing/array_creation.py:21: in <lambda>
return iter(lambda: list(itertools.islice(iterable, chunk_size)), [])
../../../.pipenv/lib/python3.11/site-packages/bw_processing/utils.py:72: in <genexpr>
data = (dictionary_formatter(row) for row in iterator)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = Brightway2 SQLiteBackend: partial_respiration_db
sql = "SELECT e.data, a.id, b.id, e.input_database, e.input_code, e.output_database, e.output_code\n FROM exc... == e.output_database\n WHERE e.output_database = ?\n AND e.type = 'biosphere'\n "
dependents = {'biosphere3'}, flip = False
def exchange_data_iterator(self, sql, dependents, flip=False):
"""Iterate over exchanges and format for ``bw_processing`` arrays.
``dependents`` is a set of dependent database names.
``flip`` means flip the numeric sign; see ``bw_processing`` docs.
Uses raw sqlite3 to retrieve data for ~2x speed boost."""
connection = sqlite3.connect(sqlite3_lci_db._filepath)
cursor = connection.cursor()
for line in cursor.execute(sql, (self.name,)):
(
data,
row,
col,
input_database,
input_code,
output_database,
output_code,
) = line
# Modify ``dependents`` in place
if input_database != output_database:
dependents.add(input_database)
data = pickle.loads(bytes(data))
check_exchange(data)
if row is None or col is None:
> raise UnknownObject(
(
"Exchange between {} and {} is invalid "
"- one of these objects is unknown (i.e. doesn't exist "
"as a process dataset)"
).format(
(input_database, input_code), (output_database, output_code)
)
)
E bw2data.errors.UnknownObject: Exchange between ('biosphere3', '9ec076d9-6d9f-4a0b-9851-730626ed4319') and ('partial_respiration_db', '7599062216496486961') is invalid - one of these objects is unknown (i.e. doesn't exist as a process dataset)
../../../.pipenv/lib/python3.11/site-packages/bw2data/backends/base.py:680: UnknownObjectI'm not sure what I'm missing here... |
|
Hi @cmutel do you think you could have a look at what the problem is? (conflicting db versions?) |
|
The test fails because when testing, there is no "biosphere3" database. The fixture must be self-sufficient |
|
@tngTUDOR thanks for that information, would you know if there is an easy way to include biosphere3? I imagine it would be better to keep these in the database for the test, right? |
|
The best would be to create a fixture to populate a project with only the required data (biosphere3 db with the flows referenced in the restored bwpackage ...). I remember having done this at some point in the past, I would need to verify how to achieve this exactly nowdays |
|
Here are the 3 biosphere3 datasets you reference to:
|
|
Here's a way to write the test that would "work":
import os
import pytest
from bw2data.tests import bw2test
from bw2data.database import DatabaseChooser
from bw2io import BW2Package
FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "bw2package")
@pytest.fixture
def mini_biosphere():
return {
("biosphere3", "9ec076d9-6d9f-4a0b-9851-730626ed4319"): {
"categories": ("air",),
"code": "9ec076d9-6d9f-4a0b-9851-730626ed4319",
"CAS number": "007782-44-7",
"synonyms": ["molecular oxygen"],
"name": "Oxygen",
"database": "biosphere3",
"unit": "kilogram",
"type": "emission",
},
("biosphere3", "14ea575b-5caa-4958-acf7-0bcc47f9cadf"): {
"categories": ("soil",),
"code": "14ea575b-5caa-4958-acf7-0bcc47f9cadf",
"CAS number": "007440-44-0",
"synonyms": [],
"name": "Carbon",
"database": "biosphere3",
"unit": "kilogram",
"type": "emission",
},
("biosphere3", "eba59fd6-f37e-41dc-9ca3-c7ea22d602c7"): {
"categories": ("air",),
"code": "eba59fd6-f37e-41dc-9ca3-c7ea22d602c7",
"CAS number": "000124-38-9",
"synonyms": ["Carbon dioxide"],
"name": "Carbon dioxide, non-fossil",
"database": "biosphere3",
"unit": "kilogram",
"type": "emission",
},
}
@bw2test
def test_bw2_compat(mini_biosphere):
mini_biosphere3_db = DatabaseChooser("biosphere3")
mini_biosphere3_db.register()
mini_biosphere3_db.write(mini_biosphere)
obj = BW2Package.import_file(os.path.join(FIXTURES, "bw2_compat_test.zip"))[0]
a = obj.get("7599062216496486961")
assert a["name"] == "partial_respiration"
assert a["unit"] == "g"
assert a["type"] == "process"
a = obj.get("3866902554231372371")
assert a["name"] == "C_inactivated"
assert a["unit"] == "g"
assert a["type"] == "production"a passing test exampleyes, but we need a test to make sure that a package created with 2.5, is re-useable again with 2.5 after the code modification proposes. Is there a test already covering the BW2Package() roundtrip ? |
What I'm saying, is that my suggestion above is only a "prototypical" idea. We must integrate this test to the packaging test file. |
|
wow, that's great, thanks for the help!
the code only changes what happens if we encounter |
|
@tngTUDOR I don't see any tests being run, do you have access to it and, if so, could you check if they can be (re)started? |
Fixes #135
As @renaud correctly analyzed, we need to map the 2 and 2.5 backend structures so the import can happen.
Note that this is important for people who don't have access to Ecoinvent at the moment and want to check the BW notebook using Forwast, for instance, as it was generated with bw2
EDIT: I imagine that bw25 package will always be bundled together so I did not check
bw2dataversion, let me know if this is necessary