From d7040e5027d97d64f0177df9ad3dccca737bae8f Mon Sep 17 00:00:00 2001 From: Angelo Romano Date: Mon, 18 Nov 2024 12:30:08 +0100 Subject: [PATCH] [#188587672]: Adjustments and enhancements --- integration/fixtures.py | 127 ++++++++++++++++++---------------------- pytest.ini | 2 +- scrunch/session.py | 22 +++++-- 3 files changed, 75 insertions(+), 76 deletions(-) diff --git a/integration/fixtures.py b/integration/fixtures.py index 5af06d6b..56fd9413 100644 --- a/integration/fixtures.py +++ b/integration/fixtures.py @@ -2,89 +2,77 @@ import os from unittest import TestCase -from scrunch.helpers import subvar_alias from scrunch import connect +from scrunch.helpers import subvar_alias class BaseIntegrationTestCase(TestCase): def setUp(self): self.host = os.environ["SCRUNCH_HOST"] + if self.host[-1] != "/": + self.host += "/" username = os.environ["SCRUNCH_USER"] password = os.environ["SCRUNCH_PASS"] - self.site = connect(username, password, self.host) - assert (self - .site is not None), "Unable to connect to %s" % self.host + api_key = os.environ.get("CRUNCH_API_KEY") or "" + self.site = connect(username, password, self.host, api_key=api_key) + assert self.site is not None, "Unable to connect to %s" % self.host # These are the categories that multiple response use. Selected and Not Selected MR_CATS = [ - {"id": 1, "name": "Selected", "missing": False, "numeric_value": None, "selected": True}, - {"id": 3, "name": "Other Selected", "missing": False, "numeric_value": None, "selected": True}, - {"id": 2, "name": "Not selected", "missing": False, "numeric_value": None, "selected": False} + { + "id": 1, + "name": "Selected", + "missing": False, + "numeric_value": None, + "selected": True, + }, + { + "id": 3, + "name": "Other Selected", + "missing": False, + "numeric_value": None, + "selected": True, + }, + { + "id": 2, + "name": "Not selected", + "missing": False, + "numeric_value": None, + "selected": False, + }, ] NEWS_DATASET = { - "caseid": { - "name": "Case ID", - "type": "numeric" - }, - "age": { - "name": "Age", - "type": 'numeric', - }, + "caseid": {"name": "Case ID", "type": "numeric"}, + "age": {"name": "Age", "type": "numeric"}, "newssource": { "name": "News source", "type": "multiple_response", "categories": MR_CATS, - "subreferences": [{ - "name": "Facebook", - "alias": "newssource_1" - }, { - "name": "Twitter", - "alias": "newssource_2" - }, { - "name": "Google news", - "alias": "newssource_3" - }, { - "name": "Reddit", - "alias": "newssource_4" - }, { - "name": "NY Times (Print)", - "alias": "newssource_5" - }, { - "name": "Washington Post (Print)", - "alias": "newssource_6" - }, { - "name": "NBC News", - "alias": "newssource_7" - }, { - "name": "NPR", - "alias": "newssource_8" - }, { - "name": "Fox", - "alias": "newssource_9" - }, { - "name": "Local radio", - "alias": "newssource_10" - }] + "subreferences": [ + {"name": "Facebook", "alias": "newssource_1"}, + {"name": "Twitter", "alias": "newssource_2"}, + {"name": "Google news", "alias": "newssource_3"}, + {"name": "Reddit", "alias": "newssource_4"}, + {"name": "NY Times (Print)", "alias": "newssource_5"}, + {"name": "Washington Post (Print)", "alias": "newssource_6"}, + {"name": "NBC News", "alias": "newssource_7"}, + {"name": "NPR", "alias": "newssource_8"}, + {"name": "Fox", "alias": "newssource_9"}, + {"name": "Local radio", "alias": "newssource_10"}, + ], }, "socialmedia": { "name": "Accounts in social media", "type": "multiple_response", "categories": MR_CATS, - "subreferences": [{ - "name": "Facebook", - "alias": "socialmedia_1" - }, { - "name": "Twitter", - "alias": "socialmedia_2" - }, { - "name": "Google+", - "alias": "socialmedia_3" - }, { - "name": "VK", - "alias": "socialmedia_4" - }] + "subreferences": [ + {"name": "Facebook", "alias": "socialmedia_1"}, + {"name": "Twitter", "alias": "socialmedia_2"}, + {"name": "Google+", "alias": "socialmedia_3"}, + {"name": "VK", "alias": "socialmedia_4"}, + ], }, "gender": { "name": "Gender", @@ -93,8 +81,8 @@ def setUp(self): {"id": 1, "name": "Female", "missing": False, "numeric_value": None}, {"id": 2, "name": "Male", "missing": False, "numeric_value": None}, {"id": -1, "name": "No Data", "missing": True, "numeric_value": None}, - ] - } + ], + }, } RECODES_CSV_OUTPUT = """newssource_1,newssource_2,newssource_3,newssource_4,newssource_5,newssource_6,newssource_7,newssource_8,newssource_9,newssource_10,age,socialmedia_1,socialmedia_2,socialmedia_3,socialmedia_4,caseid,gender,agerange,origintype_1,origintype_2,origintype_3,origintype_4,origintype_copy_1,origintype_copy_2,origintype_copy_3,origintype_copy_4,onlinenewssource_1,onlinenewssource_2,over35 @@ -122,7 +110,7 @@ def setUp(self): "socialmedia_2": [1, 2, 1, 1, 2, 1, 2], "socialmedia_3": [2, 2, 1, 2, 2, 1, 2], "socialmedia_4": [2, 2, 1, 2, 2, 2, 2], - "gender": [1, 2, 2, 1, 1, 1, 2] + "gender": [1, 2, 2, 1, 1, 1, 2], } @@ -131,14 +119,13 @@ def mr_in(ds, mr_alias, subvars): Temporary helper until scrunch can parse correctly the expression: mr.has_any([sv1, sv2...]) """ - variables = ds.resource.variables.by('alias') + variables = ds.resource.variables.by("alias") mr = variables[mr_alias].entity - subvariables = mr.subvariables.by('alias') + subvariables = mr.subvariables.by("alias") return { - 'function': 'any', - 'args': [{ - 'variable': mr.self - }, { - 'column': [subvariables[subvar_alias(mr_alias, sv)].id for sv in subvars] - }] + "function": "any", + "args": [ + {"variable": mr.self}, + {"column": [subvariables[subvar_alias(mr_alias, sv)].id for sv in subvars]}, + ], } diff --git a/pytest.ini b/pytest.ini index 1d5f99a7..051ddd8d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] norecursedirs=dist build .tox .eggs examples -addopts=--doctest-modules --cov=scrunch --cov-config=.coveragerc -p no:sugar +addopts=--doctest-modules -p no:sugar doctest_optionflags=ALLOW_UNICODE ELLIPSIS diff --git a/scrunch/session.py b/scrunch/session.py index be8198e1..a418be34 100644 --- a/scrunch/session.py +++ b/scrunch/session.py @@ -1,9 +1,18 @@ -from pycrunch.version import __version__ as pycrunch_version +import os + from pycrunch import connect as _connect from pycrunch.elements import ElementSession +from pycrunch.version import __version__ as pycrunch_version from .version import __version__ +SSL_UNSAFE = os.environ.get("SSL_UNSAFE", "").strip().lower() in ( + "1", + "true", + "yes", + "y", +) + class ScrunchSession(ElementSession): headers = { @@ -16,6 +25,7 @@ class ScrunchSSLUnsafeSession(ScrunchSession): A subclass of `ScrunchSession` that skips SSL certificate validation when trying to connect to the API server. Useful for local testing. """ + def __init__(self, *args, **kwargs): super(ScrunchSSLUnsafeSession, self).__init__(*args, **kwargs) self.verify = False @@ -26,15 +36,17 @@ def __init__(self, *args, **kwargs): def set_feature_flags(site): feature_flags = { - flag_name: site.follow('feature_flag', - 'feature_name=%s' % flag_name).value['active'] + flag_name: site.follow("feature_flag", "feature_name=%s" % flag_name).value[ + "active" + ] for flag_name in FLAGS_TO_CHECK } - setattr(site.session, 'feature_flags', feature_flags) + setattr(site.session, "feature_flags", feature_flags) return site def connect(*args, **kwargs): - _site = _connect(session_class=ScrunchSession, *args, **kwargs) + session_class = ScrunchSSLUnsafeSession if SSL_UNSAFE else ScrunchSession + _site = _connect(session_class=session_class, *args, **kwargs) _site = set_feature_flags(_site) return _site