Implement new version of FastRun

LeMyst · Jun 7, 2022 · 1b16c78 · 1b16c78
1 parent 92a8a3d
commit 1b16c78
Show file tree

Hide file tree

Showing 32 changed files with 779 additions and 883 deletions.
diff --git a/.idea/inspectionProfiles/WikibaseIntegrator.xml b/.idea/inspectionProfiles/WikibaseIntegrator.xml
diff --git a/README.md b/README.md
@@ -567,7 +567,15 @@ for entrez_id, ensembl in raw_data.items():
 
 Note: Fastrun mode checks for equality of property/value pairs, qualifiers (not including qualifier attributes), labels,
 aliases and description, but it ignores references by default!
-References can be checked in fast run mode by setting `use_refs` to `True`.
+References can be checked in fast run mode by setting `use_references` to `True`.
+
+# Statistics #
+
+| Dataset                     | partial fastrun | fastrun without qualifiers/references | fastrun with qualifiers | fastrun with qualifiers/references |
+|:----------------------------|----------------:|--------------------------------------:|------------------------:|-----------------------------------:|
+| Communes (34990 elements)   |               ? |                                  7min |                     30s |                                60s |
+| Cantons (2042 elements)     |               ? |                                     ? |                       ? |                                  ? |
+| Départements (100 elements) |           70min |                                    1s |                     30s |                                60s |
 
 # Debugging #
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -34,3 +34,4 @@ disable = [
 
 [tool.pytest.ini_options]
 log_cli = true
+log_cli_level = 'DEBUG'
diff --git a/test/test_all.py b/test/test_all.py
@@ -1,12 +1,11 @@
 import copy
+import logging
 import unittest
 
 from wikibaseintegrator import WikibaseIntegrator, datatypes, wbi_fastrun
 from wikibaseintegrator.datatypes import BaseDataType, Item
-from wikibaseintegrator.entities import ItemEntity
+from wikibaseintegrator.entities import BaseEntity, ItemEntity
 from wikibaseintegrator.wbi_config import config as wbi_config
-from wikibaseintegrator.wbi_enums import ActionIfExists
-from wikibaseintegrator.wbi_fastrun import get_fastrun_container
 
 wbi_config['USER_AGENT'] = 'WikibaseIntegrator-pytest/1.0 (test_all.py)'
 
@@ -53,66 +52,24 @@ class TestFastRun(unittest.TestCase):
     """
     some basic tests for fastrun mode
     """
+    logging.basicConfig(level=logging.DEBUG)
 
     def test_fastrun(self):
         statements = [
-            datatypes.ExternalID(value='P40095', prop_nr='P352'),
+            datatypes.ExternalID(value='A0A023PZB3', prop_nr='P352'),
             datatypes.ExternalID(value='YER158C', prop_nr='P705')
         ]
 
         frc = wbi_fastrun.FastRunContainer(base_filter=[BaseDataType(prop_nr='P352'), datatypes.Item(prop_nr='P703', value='Q27510868')], base_data_type=datatypes.BaseDataType)
 
-        fastrun_result = frc.write_required(data=statements)
+        entity = BaseEntity().add_claims(statements)
 
-        if fastrun_result:
-            message = 'fastrun failed'
-        else:
-            message = 'successful fastrun'
+        fastrun_result = frc.write_required(entity=entity)
 
         # here, fastrun should succeed, if not, test failed
         if fastrun_result:
             raise ValueError
 
-    def test_fastrun_label(self):
-        # tests fastrun label, description and aliases, and label in another language
-        frc = get_fastrun_container(base_filter=[datatypes.ExternalID(value='/m/02j71', prop_nr='P646')])
-        item = WikibaseIntegrator().item.get('Q2')
-
-        assert item.labels.get(language='en') == "Earth"
-        descr = item.descriptions.get(language='en')
-        assert len(descr) > 3
-        assert "the Earth" in item.aliases.get()
-
-        assert list(frc.get_language_data("Q2", 'en', 'label'))[0] == item.labels.get(language='en')
-        assert frc.check_language_data("Q2", ['not the Earth'], 'en', 'label')
-        assert "the Earth" in item.aliases.get()
-        assert "planet" in item.descriptions.get()
-
-        assert item.labels.get('es') == "Tierra"
-
-        item.descriptions.set(value=descr)
-        item.descriptions.set(value="fghjkl")
-        assert item.get_json()['descriptions']['en'] == {'language': 'en', 'value': 'fghjkl'}
-        item.labels.set(value="Earth")
-        item.labels.set(value="xfgfdsg")
-        assert item.get_json()['labels']['en'] == {'language': 'en', 'value': 'xfgfdsg'}
-        item.aliases.set(values=["fake alias"], action_if_exists=ActionIfExists.APPEND)
-        assert {'language': 'en', 'value': 'fake alias'} in item.get_json()['aliases']['en']
-
-        # something that's empty (for now.., can change, so this just makes sure no exception is thrown)
-        frc.check_language_data("Q2", ['Ewiase'], 'ak', 'label')
-        frc.check_language_data("Q2", ['not Ewiase'], 'ak', 'label')
-        frc.check_language_data("Q2", [''], 'ak', 'description')
-        frc.check_language_data("Q2", [], 'ak', 'aliases')
-        frc.check_language_data("Q2", ['sdf', 'sdd'], 'ak', 'aliases')
-
-        item.labels.get(language="ak")
-        item.descriptions.get(language='ak')
-        item.aliases.get(language="ak")
-        item.labels.set(value="label", language="ak")
-        item.descriptions.set(value="d", language="ak")
-        item.aliases.set(values=["a"], language="ak", action_if_exists=ActionIfExists.APPEND)
-
 
 def test_sitelinks():
     item = wbi.item.get('Q622901')

diff --git a/test/test_entity_item.py b/test/test_entity_item.py
@@ -48,15 +48,15 @@ def test_write(self):
     def test_write_not_required(self):
         assert not wbi.item.get('Q582').write_required(base_filter=[BaseDataType(prop_nr='P1791')])
 
+    def test_write_not_required_ref(self):
+        assert not wbi.item.get('Q582').write_required(base_filter=[BaseDataType(prop_nr='P1464')], use_references=True)
+
     def test_write_required(self):
         item = wbi.item.get('Q582')
         item.claims.add(Item(prop_nr='P1791', value='Q42'))
         assert item.write_required([BaseDataType(prop_nr='P1791')])
 
-    def test_write_not_required_ref(self):
-        assert not wbi.item.get('Q582').write_required(base_filter=[BaseDataType(prop_nr='P2581')], use_refs=True)
-
     def test_write_required_ref(self):
         item = wbi.item.get('Q582')
-        item.claims.get('P2581')[0].references.references.pop()
-        assert item.write_required(base_filter=[BaseDataType(prop_nr='P2581')], use_refs=True)
+        item.claims.get('P1464')[0].references.references.pop()
+        assert item.write_required(base_filter=[BaseDataType(prop_nr='P1464')], use_references=True)
diff --git a/test/test_wbi_core.py b/test/test_wbi_core.py
@@ -231,7 +231,7 @@ def test_new_item_creation(self):
             MonolingualText(text="xxx", language="fr", prop_nr="P7"),
             Quantity(amount=-5.04, prop_nr="P8"),
             Quantity(amount=5.06, upper_bound=9.99, lower_bound=-2.22, unit="Q11573", prop_nr="P8"),
-            CommonsMedia(value="xxx", prop_nr="P9"),
+            CommonsMedia(value="Place lazare goujon.jpg", prop_nr="P9"),
             GlobeCoordinate(latitude=1.2345, longitude=-1.2345, precision=12, prop_nr="P10"),
             GeoShape(value="Data:xxx.map", prop_nr="P11"),
             Property(value="P123", prop_nr="P12"),

diff --git a/test/test_wbi_fastrun.py b/test/test_wbi_fastrun.py