diff --git a/mig/install/__init__.py b/mig/install/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mig/install/generateconfs.py b/mig/install/generateconfs.py index 098d17188..eb3e097dd 100755 --- a/mig/install/generateconfs.py +++ b/mig/install/generateconfs.py @@ -48,7 +48,7 @@ # NOTE: moved mig imports into try/except to avoid autopep8 moving to top! try: from mig.shared.defaults import MIG_BASE, MIG_ENV - from mig.shared.install import generate_confs + from mig.shared.install import generate_confs, _GENERATE_CONFS_PARAMETERS except ImportError: print("ERROR: the migrid modules must be in PYTHONPATH") sys.exit(1) @@ -65,7 +65,7 @@ def usage(options): ''' % (sys.argv[0], '\n'.join(lines))) -def main(argv, _generate_confs=generate_confs, _print=print): +def _make_parameters(): str_names = [ 'source', 'destination', @@ -247,6 +247,8 @@ def main(argv, _generate_confs=generate_confs, _print=print): 'openid_port', 'openid_show_port', 'openid_session_lifetime', + 'seafile_secret', + 'seafile_ccnetid', 'seafile_seahub_port', 'seafile_seafhttp_port', 'seafile_client_port', @@ -303,7 +305,15 @@ def main(argv, _generate_confs=generate_confs, _print=print): 'io_account_expire', 'gdp_email_notify', ] - names = str_names + int_names + bool_names + return (str_names, int_names, bool_names) + + +str_names, int_names, bool_names = _make_parameters() +_PARAMETERS = str_names + int_names + bool_names + + +def main(argv, _generate_confs=generate_confs, _print=print): + names = _PARAMETERS settings, options, result = {}, {}, {} default_val = 'DEFAULT' # Force values to expected type diff --git a/mig/shared/compat.py b/mig/shared/compat.py index 01069ce43..0c0f46334 100644 --- a/mig/shared/compat.py +++ b/mig/shared/compat.py @@ -34,6 +34,7 @@ from past.builtins import basestring import codecs +import inspect import io import sys # NOTE: StringIO is only available in python2 @@ -93,6 +94,15 @@ def ensure_native_string(string_or_bytes): return textual_output +def inspect_args(func): + """Wrapper to return the arguments of a function.""" + + if PY2: + return inspect.getargspec(func).args + else: + return inspect.getfullargspec(func).args + + def NativeStringIO(initial_value=''): """Mock StringIO pseudo-class to create a StringIO matching the native string coding form. That is a BytesIO with utf8 on python 2 and unicode diff --git a/mig/shared/install.py b/mig/shared/install.py index 0d916e911..2f537ce19 100644 --- a/mig/shared/install.py +++ b/mig/shared/install.py @@ -50,15 +50,16 @@ import sys from mig.shared.base import force_native_str, force_utf8 +from mig.shared.compat import ensure_native_string, inspect_args, \ + SimpleNamespace from mig.shared.defaults import default_http_port, default_https_port, \ auth_openid_mig_db, auth_openid_ext_db, MIG_BASE, STRONG_TLS_CIPHERS, \ STRONG_TLS_CURVES, STRONG_SSH_HOSTKEYALGOS, STRONG_SSH_KEXALGOS, \ STRONG_SSH_CIPHERS, STRONG_SSH_MACS, LEGACY_SSH_HOSTKEYALGOS, \ LEGACY_SSH_KEXALGOS, LEGACY_SSH_CIPHERS, LEGACY_SSH_MACS, \ FALLBACK_SSH_HOSTKEYALGOS, FALLBACK_SSH_KEXALGOS, FALLBACK_SSH_CIPHERS, \ - FALLBACK_SSH_MACS, CRACK_USERNAME_REGEX, CRACK_WEB_REGEX, keyword_any, \ - keyword_auto -from mig.shared.compat import ensure_native_string + FALLBACK_SSH_MACS, CRACK_USERNAME_REGEX, CRACK_WEB_REGEX, \ + keyword_any, keyword_auto from mig.shared.fileio import read_file, read_file_lines, write_file, \ write_file_lines from mig.shared.htmlgen import menu_items @@ -283,16 +284,7 @@ def template_remove(template_file, remove_pattern): ] -def generate_confs( - generateconfs_output_path, - # NOTE: make sure command line args with white-space are properly wrapped - generateconfs_command=subprocess.list2cmdline(sys.argv), - source=keyword_auto, - destination=keyword_auto, - user=keyword_auto, - group=keyword_auto, - timezone=keyword_auto, - destination_suffix="", +_DEFAULTS = SimpleNamespace( base_fqdn='', public_fqdn='', public_alias_fqdn='', @@ -520,6 +512,246 @@ def generate_confs( datasafety_link='', datasafety_text='', wwwserve_max_bytes=-1, +) + + +def generate_confs( + generateconfs_output_path, + # NOTE: make sure command line args with white-space are properly wrapped + generateconfs_command=subprocess.list2cmdline(sys.argv), + source=keyword_auto, + destination=keyword_auto, + user=keyword_auto, + group=keyword_auto, + timezone=keyword_auto, + destination_suffix="", + base_fqdn=_DEFAULTS.base_fqdn, + public_fqdn=_DEFAULTS.public_fqdn, + public_alias_fqdn=_DEFAULTS.public_alias_fqdn, + public_sec_fqdn=_DEFAULTS.public_sec_fqdn, + status_alias_fqdn=_DEFAULTS.status_alias_fqdn, + mig_cert_fqdn=_DEFAULTS.mig_cert_fqdn, + ext_cert_fqdn=_DEFAULTS.ext_cert_fqdn, + mig_oid_fqdn=_DEFAULTS.mig_oid_fqdn, + ext_oid_fqdn=_DEFAULTS.ext_oid_fqdn, + mig_oidc_fqdn=_DEFAULTS.mig_oidc_fqdn, + ext_oidc_fqdn=_DEFAULTS.ext_oidc_fqdn, + sid_fqdn=_DEFAULTS.sid_fqdn, + io_fqdn=_DEFAULTS.io_fqdn, + cert_fqdn_extras=_DEFAULTS.cert_fqdn_extras, + cloud_fqdn=_DEFAULTS.cloud_fqdn, + seafile_fqdn=_DEFAULTS.seafile_fqdn, + seafile_base=_DEFAULTS.seafile_base, + seafmedia_base=_DEFAULTS.seafmedia_base, + seafhttp_base=_DEFAULTS.seafhttp_base, + openid_address=_DEFAULTS.openid_address, + sftp_address=_DEFAULTS.sftp_address, + sftp_subsys_address=_DEFAULTS.sftp_subsys_address, + ftps_address=_DEFAULTS.ftps_address, + davs_address=_DEFAULTS.davs_address, + jupyter_services=_DEFAULTS.jupyter_services, + jupyter_services_desc=_DEFAULTS.jupyter_services_desc, + cloud_services=_DEFAULTS.cloud_services, + cloud_services_desc=_DEFAULTS.cloud_services_desc, + apache_version=_DEFAULTS.apache_version, + apache_etc=_DEFAULTS.apache_etc, + apache_run=_DEFAULTS.apache_run, + apache_lock=_DEFAULTS.apache_lock, + apache_log=_DEFAULTS.apache_log, + apache_worker_procs=_DEFAULTS.apache_worker_procs, + openssh_version=_DEFAULTS.openssh_version, + mig_code=_DEFAULTS.mig_code, + mig_state=_DEFAULTS.mig_state, + mig_certs=_DEFAULTS.mig_certs, + auto_add_cert_user=_DEFAULTS.auto_add_cert_user, + auto_add_oid_user=_DEFAULTS.auto_add_oid_user, + auto_add_oidc_user=_DEFAULTS.auto_add_oidc_user, + auto_add_filter_fields=_DEFAULTS.auto_add_filter_fields, + auto_add_filter_method=_DEFAULTS.auto_add_filter_method, + auto_add_user_permit=_DEFAULTS.auto_add_user_permit, + auto_add_user_with_peer=_DEFAULTS.auto_add_user_with_peer, + cert_valid_days=_DEFAULTS.cert_valid_days, + oid_valid_days=_DEFAULTS.oid_valid_days, + oidc_valid_days=_DEFAULTS.oidc_valid_days, + generic_valid_days=_DEFAULTS.generic_valid_days, + enable_migadmin=_DEFAULTS.enable_migadmin, + enable_sftp=_DEFAULTS.enable_sftp, + enable_sftp_subsys=_DEFAULTS.enable_sftp_subsys, + sftp_subsys_auth_procs=_DEFAULTS.sftp_subsys_auth_procs, + enable_davs=_DEFAULTS.enable_davs, + enable_ftps=_DEFAULTS.enable_ftps, + enable_wsgi=_DEFAULTS.enable_wsgi, + wsgi_procs=_DEFAULTS.wsgi_procs, + enable_gdp=_DEFAULTS.enable_gdp, + enable_jobs=_DEFAULTS.enable_jobs, + enable_resources=_DEFAULTS.enable_resources, + enable_workflows=_DEFAULTS.enable_workflows, + enable_events=_DEFAULTS.enable_events, + enable_sharelinks=_DEFAULTS.enable_sharelinks, + enable_transfers=_DEFAULTS.enable_transfers, + enable_freeze=_DEFAULTS.enable_freeze, + enable_sandboxes=_DEFAULTS.enable_sandboxes, + enable_vmachines=_DEFAULTS.enable_vmachines, + enable_preview=_DEFAULTS.enable_preview, + enable_jupyter=_DEFAULTS.enable_jupyter, + enable_cloud=_DEFAULTS.enable_cloud, + enable_hsts=_DEFAULTS.enable_hsts, + enable_vhost_certs=_DEFAULTS.enable_vhost_certs, + enable_verify_certs=_DEFAULTS.enable_verify_certs, + enable_seafile=_DEFAULTS.enable_seafile, + enable_duplicati=_DEFAULTS.enable_duplicati, + enable_crontab=_DEFAULTS.enable_crontab, + enable_notify=_DEFAULTS.enable_notify, + enable_imnotify=_DEFAULTS.enable_imnotify, + enable_dev_accounts=_DEFAULTS.enable_dev_accounts, + enable_twofactor=_DEFAULTS.enable_twofactor, + twofactor_mandatory_protos=_DEFAULTS.twofactor_mandatory_protos, + enable_twofactor_strict_address=_DEFAULTS.enable_twofactor_strict_address, + twofactor_auth_apps=_DEFAULTS.twofactor_auth_apps, + enable_peers=_DEFAULTS.enable_peers, + peers_mandatory=_DEFAULTS.peers_mandatory, + peers_explicit_fields=_DEFAULTS.peers_explicit_fields, + peers_contact_hint=_DEFAULTS.peers_contact_hint, + enable_cracklib=_DEFAULTS.enable_cracklib, + enable_openid=_DEFAULTS.enable_openid, + enable_gravatars=_DEFAULTS.enable_gravatars, + enable_sitestatus=_DEFAULTS.enable_sitestatus, + enable_quota=_DEFAULTS.enable_quota, + prefer_python3=_DEFAULTS.prefer_python3, + io_account_expire=_DEFAULTS.io_account_expire, + gdp_email_notify=_DEFAULTS.gdp_email_notify, + user_interface=_DEFAULTS.user_interface, + mig_oid_title=_DEFAULTS.mig_oid_title, + mig_oid_provider=_DEFAULTS.mig_oid_provider, + ext_oid_title=_DEFAULTS.ext_oid_title, + ext_oid_provider=_DEFAULTS.ext_oid_provider, + mig_oidc_title=_DEFAULTS.mig_oidc_title, + mig_oidc_provider_meta_url=_DEFAULTS.mig_oidc_provider_meta_url, + ext_oidc_title=_DEFAULTS.ext_oidc_title, + ext_oidc_provider_meta_url=_DEFAULTS.ext_oidc_provider_meta_url, + ext_oidc_provider_issuer=_DEFAULTS.ext_oidc_provider_issuer, + ext_oidc_provider_authorization_endpoint=_DEFAULTS.ext_oidc_provider_authorization_endpoint, + ext_oidc_provider_verify_cert_files=_DEFAULTS.ext_oidc_provider_verify_cert_files, + ext_oidc_provider_token_endpoint=_DEFAULTS.ext_oidc_provider_token_endpoint, + ext_oidc_provider_token_endpoint_auth=_DEFAULTS.ext_oidc_provider_token_endpoint_auth, + ext_oidc_provider_user_info_endpoint=_DEFAULTS.ext_oidc_provider_user_info_endpoint, + ext_oidc_scope=_DEFAULTS.ext_oidc_scope, + ext_oidc_user_info_token_method=_DEFAULTS.ext_oidc_user_info_token_method, + ext_oidc_public_key_files=_DEFAULTS.ext_oidc_public_key_files, + ext_oidc_private_key_files=_DEFAULTS.ext_oidc_private_key_files, + ext_oidc_response_type=_DEFAULTS.ext_oidc_response_type, + ext_oidc_response_mode=_DEFAULTS.ext_oidc_response_mode, + ext_oidc_client_id=_DEFAULTS.ext_oidc_client_id, + ext_oidc_client_name=_DEFAULTS.ext_oidc_client_name, + ext_oidc_pkce_method=_DEFAULTS.ext_oidc_pkce_method, + ext_oidc_id_token_encrypted_response_alg=_DEFAULTS.ext_oidc_id_token_encrypted_response_alg, + ext_oidc_id_token_encrypted_response_enc=_DEFAULTS.ext_oidc_id_token_encrypted_response_enc, + ext_oidc_user_info_signed_response_alg=_DEFAULTS.ext_oidc_user_info_signed_response_alg, + ext_oidc_cookie_same_site=_DEFAULTS.ext_oidc_cookie_same_site, + ext_oidc_pass_cookies=_DEFAULTS.ext_oidc_pass_cookies, + ext_oidc_remote_user_claim=_DEFAULTS.ext_oidc_remote_user_claim, + ext_oidc_pass_claim_as=_DEFAULTS.ext_oidc_pass_claim_as, + ext_oidc_rewrite_cookie=_DEFAULTS.ext_oidc_rewrite_cookie, + dhparams_path=_DEFAULTS.dhparams_path, + daemon_keycert=_DEFAULTS.daemon_keycert, + daemon_keycert_sha256=_DEFAULTS.daemon_keycert_sha256, + daemon_pubkey=_DEFAULTS.daemon_pubkey, + daemon_pubkey_from_dns=_DEFAULTS.daemon_pubkey_from_dns, + daemon_pubkey_md5=_DEFAULTS.daemon_pubkey_md5, + daemon_pubkey_sha256=_DEFAULTS.daemon_pubkey_sha256, + daemon_show_address=_DEFAULTS.daemon_show_address, + alias_field=_DEFAULTS.alias_field, + peers_permit=_DEFAULTS.peers_permit, + vgrid_creators=_DEFAULTS.vgrid_creators, + vgrid_managers=_DEFAULTS.vgrid_managers, + signup_methods=_DEFAULTS.signup_methods, + login_methods=_DEFAULTS.login_methods, + digest_salt=_DEFAULTS.digest_salt, + crypto_salt=_DEFAULTS.crypto_salt, + csrf_protection=_DEFAULTS.csrf_protection, + password_policy=_DEFAULTS.password_policy, + password_legacy_policy=_DEFAULTS.password_legacy_policy, + hg_path=_DEFAULTS.hg_path, + hgweb_scripts=_DEFAULTS.hgweb_scripts, + trac_admin_path=_DEFAULTS.trac_admin_path, + trac_ini_path=_DEFAULTS.trac_ini_path, + public_port=_DEFAULTS.public_port, + public_http_port=_DEFAULTS.public_http_port, + public_https_port=_DEFAULTS.public_https_port, + mig_cert_port=_DEFAULTS.mig_cert_port, + ext_cert_port=_DEFAULTS.ext_cert_port, + mig_oid_port=_DEFAULTS.mig_oid_port, + ext_oid_port=_DEFAULTS.ext_oid_port, + mig_oidc_port=_DEFAULTS.mig_oidc_port, + ext_oidc_port=_DEFAULTS.ext_oidc_port, + sid_port=_DEFAULTS.sid_port, + sftp_port=_DEFAULTS.sftp_port, + sftp_subsys_port=_DEFAULTS.sftp_subsys_port, + sftp_show_port=_DEFAULTS.sftp_show_port, + sftp_max_sessions=_DEFAULTS.sftp_max_sessions, + davs_port=_DEFAULTS.davs_port, + davs_show_port=_DEFAULTS.davs_show_port, + ftps_ctrl_port=_DEFAULTS.ftps_ctrl_port, + ftps_ctrl_show_port=_DEFAULTS.ftps_ctrl_show_port, + ftps_pasv_ports=_DEFAULTS.ftps_pasv_ports, + openid_port=_DEFAULTS.openid_port, + openid_show_port=_DEFAULTS.openid_show_port, + openid_session_lifetime=_DEFAULTS.openid_session_lifetime, + seafile_secret=_DEFAULTS.seafile_secret, + seafile_ccnetid=_DEFAULTS.seafile_ccnetid, + seafile_seahub_port=_DEFAULTS.seafile_seahub_port, + seafile_seafhttp_port=_DEFAULTS.seafile_seafhttp_port, + seafile_client_port=_DEFAULTS.seafile_client_port, + seafile_quota=_DEFAULTS.seafile_quota, + seafile_ro_access=_DEFAULTS.seafile_ro_access, + public_use_https=_DEFAULTS.public_use_https, + user_clause=_DEFAULTS.user_clause, + group_clause=_DEFAULTS.group_clause, + listen_clause=_DEFAULTS.listen_clause, + serveralias_clause=_DEFAULTS.serveralias_clause, + distro=_DEFAULTS.distro, + autolaunch_page=_DEFAULTS.autolaunch_page, + landing_page=_DEFAULTS.landing_page, + skin=_DEFAULTS.skin, + title=_DEFAULTS.title, + short_title=_DEFAULTS.short_title, + extra_userpage_scripts=_DEFAULTS.extra_userpage_scripts, + extra_userpage_styles=_DEFAULTS.extra_userpage_styles, + external_doc=_DEFAULTS.external_doc, + vgrid_label=_DEFAULTS.vgrid_label, + secscan_addr=_DEFAULTS.secscan_addr, + default_menu=_DEFAULTS.default_menu, + user_menu=_DEFAULTS.user_menu, + collaboration_links=_DEFAULTS.collaboration_links, + default_vgrid_links=_DEFAULTS.default_vgrid_links, + advanced_vgrid_links=_DEFAULTS.advanced_vgrid_links, + support_email=_DEFAULTS.support_email, + admin_email=_DEFAULTS.admin_email, + admin_list=_DEFAULTS.admin_list, + smtp_server=_DEFAULTS.smtp_server, + smtp_sender=_DEFAULTS.smtp_sender, + permanent_freeze=_DEFAULTS.permanent_freeze, + log_level=_DEFAULTS.log_level, + freeze_to_tape=_DEFAULTS.freeze_to_tape, + status_system_match=_DEFAULTS.status_system_match, + storage_protocols=_DEFAULTS.storage_protocols, + duplicati_protocols=_DEFAULTS.duplicati_protocols, + imnotify_address=_DEFAULTS.imnotify_address, + imnotify_channel=_DEFAULTS.imnotify_channel, + imnotify_username=_DEFAULTS.imnotify_username, + imnotify_password=_DEFAULTS.imnotify_password, + gdp_data_categories=_DEFAULTS.gdp_data_categories, + gdp_id_scramble=_DEFAULTS.gdp_id_scramble, + gdp_path_scramble=_DEFAULTS.gdp_path_scramble, + quota_backend=_DEFAULTS.quota_backend, + quota_user_limit=_DEFAULTS.quota_user_limit, + quota_vgrid_limit=_DEFAULTS.quota_vgrid_limit, + ca_fqdn=_DEFAULTS.ca_fqdn, + ca_user=_DEFAULTS.ca_user, + ca_smtp=_DEFAULTS.ca_smtp, + datasafety_link=_DEFAULTS.datasafety_link, + datasafety_text=_DEFAULTS.datasafety_text, + wwwserve_max_bytes=_DEFAULTS.wwwserve_max_bytes, _getpwnam=pwd.getpwnam, _prepare=None, _writefiles=None, @@ -612,6 +844,9 @@ def generate_confs( return (options, user_dict) +_GENERATE_CONFS_PARAMETERS = set(inspect_args(generate_confs)) - set(_GENERATE_CONFS_NOFORWARD_KEYS) + + def _generate_confs_prepare( options, # forwarded arguments diff --git a/tests/test_mig_install_generateconfs.py b/tests/test_mig_install_generateconfs.py index e8ed90241..4590b5d58 100644 --- a/tests/test_mig_install_generateconfs.py +++ b/tests/test_mig_install_generateconfs.py @@ -35,6 +35,8 @@ from tests.support import MIG_BASE, MigTestCase, testmain, cleanpath +from mig.install.generateconfs import _PARAMETERS +from mig.shared.install import generate_confs, _GENERATE_CONFS_PARAMETERS def _import_generateconfs(): """Internal helper to work around non-package import location""" @@ -65,6 +67,13 @@ def noop(*args): pass +class MigInstallGenerateconfs(MigTestCase): + def test_consistent_parameters(self): + mismatched = _GENERATE_CONFS_PARAMETERS - set(_PARAMETERS) + self.assertEqual(len(mismatched), 0, + "defined parameters do not match generate_confs()") + + class MigInstallGenerateconfs__main(MigTestCase): """Unit test helper for the migrid code pointed to in class name""" diff --git a/tests/test_mig_shared_install.py b/tests/test_mig_shared_install.py index 5c7e567a1..acd10dd6d 100644 --- a/tests/test_mig_shared_install.py +++ b/tests/test_mig_shared_install.py @@ -39,8 +39,9 @@ from tests.support import MIG_BASE, TEST_OUTPUT_DIR, MigTestCase, \ testmain, temppath, cleanpath, fixturepath, is_path_within +from mig.install.generateconfs import _GENERATE_CONFS_PARAMETERS from mig.shared.defaults import keyword_auto -from mig.shared.install import determine_timezone, generate_confs +from mig.shared.install import determine_timezone, generate_confs, _DEFAULTS class DummyPwInfo: @@ -64,6 +65,14 @@ def noop(*args, **kwargs): return None +class MigSharedInstall(MigTestCase): + def test_consistent_parameters(self): + install_defaults_keys = set(_DEFAULTS.__dict__.keys()) + mismatched = _GENERATE_CONFS_PARAMETERS - install_defaults_keys + self.assertEqual(len(mismatched), 0, + "install defaults do not match generate_confs()") + + class MigSharedInstall__determine_timezone(MigTestCase): """Coverage of timezone determination."""