Merge pull request apache#728 from RealImpactAnalytics/svv_fernet

Fernet parameter missing in config + improving config creation

CONTRIBUTING.md

@@ -116,13 +116,13 @@ Once your unit test environment is setup, you should be able to simply run
 For example, in order to just execute the "core" unit tests, run the following: 
 
 ```
-./run_unit_tests.sh tests.core:CoreTest
+./run_unit_tests.sh tests.core:CoreTest -s
 ```
 
 or a single test method: 
 
 ``` 
-./run_unit_tests.sh tests.core:CoreTest.test_check_operators
+./run_unit_tests.sh tests.core:CoreTest.test_check_operators -s
 ```
 
 For more information on how to run a subset of the tests, take a look at the nosetests docs.

airflow/bin/cli.py

@@ -138,7 +138,7 @@ def run(args):
     if not args.pickle:
         dagbag = DagBag(subdir)
         if args.dag_id not in dagbag.dags:
-            msg = 'DAG [{0}] could not be found'.format(args.dag_id)
+            msg = 'DAG [{0}] could not be found in {1}'.format(args.dag_id, subdir)
             logging.error(msg)
             raise AirflowException(msg)
         dag = dagbag.dags[args.dag_id]

airflow/configuration.py

@@ -4,15 +4,13 @@
 from __future__ import unicode_literals
 
 from future import standard_library
+
 standard_library.install_aliases()
 from builtins import str
 from configparser import ConfigParser
 import errno
 import logging
 import os
-import sys
-import textwrap
-
 
 try:
     from cryptography.fernet import Fernet
@@ -287,6 +285,7 @@ class AirflowConfigException(Exception):
 load_examples = True
 donot_pickle = False
 dag_concurrency = 16
+fernet_key = {FERNET_KEY}
 
 [webserver]
 base_url = http://localhost:8080
@@ -396,24 +395,33 @@ def mkdir_p(path):
 else:
     AIRFLOW_CONFIG = expand_env_var(os.environ['AIRFLOW_CONFIG'])
 
-if not os.path.isfile(AIRFLOW_CONFIG):
+
+def parameterized_config(template):
     """
-    These configuration options are used to generate a default configuration
-    when it is missing. The right way to change your configuration is to alter
-    your configuration file, not this code.
+    Generates a configuration from the provided template + variables defined in
+    current scope
+    :param template: a config content templated with {{variables}}
     """
     FERNET_KEY = generate_fernet_key()
-    logging.info("Creating new config file in: " + AIRFLOW_CONFIG)
-    f = open(AIRFLOW_CONFIG, 'w')
-    f.write(DEFAULT_CONFIG.format(**locals()))
-    f.close()
+    all_vars = {k: v for d in [globals(), locals()] for k, v in d.items()}
+    return template.format(**all_vars)
 
 TEST_CONFIG_FILE = AIRFLOW_HOME + '/unittests.cfg'
 if not os.path.isfile(TEST_CONFIG_FILE):
-    logging.info("Creating new config file in: " + TEST_CONFIG_FILE)
-    f = open(TEST_CONFIG_FILE, 'w')
-    f.write(TEST_CONFIG.format(**locals()))
-    f.close()
+    logging.info("Creating new airflow config file for unit tests in: " +
+                 TEST_CONFIG_FILE)
+    with open(AIRFLOW_CONFIG, 'w') as f:
+        f.write(parameterized_config(TEST_CONFIG))
+
+if not os.path.isfile(AIRFLOW_CONFIG):
+    """
+    These configuration options are used to generate a default configuration
+    when it is missing. The right way to change your configuration is to alter
+    your configuration file, not this code.
+    """
+    logging.info("Creating new airflow config file in: " + AIRFLOW_CONFIG)
+    with open(AIRFLOW_CONFIG, 'w') as f:
+        f.write(parameterized_config(DEFAULT_CONFIG))
 
 logging.info("Reading the config from " + AIRFLOW_CONFIG)
 
@@ -437,9 +445,18 @@ def getboolean(section, key):
 def getfloat(section, key):
     return conf.getfloat(section, key)
 
+
 def getint(section, key):
     return conf.getint(section, key)
 
+
 def has_option(section, key):
     return conf.has_option(section, key)
 
+
+########################
+# convenience method to access config entries
+
+def get_dags_folder():
+    return os.path.expanduser(get('core', 'DAGS_FOLDER'))
+

run_unit_tests.sh

@@ -6,12 +6,13 @@ export AIRFLOW_CONFIG=$AIRFLOW_HOME/unittests.cfg
 # any argument received is overriding the default nose execution arguments: 
 
 nose_args=$@
-if [ -a $nose_args ]; then
+if [ -z "$nose_args" ]; then
   nose_args="--with-coverage \
 --cover-erase \
 --cover-html \
 --cover-package=airflow \
 --cover-html-dir=airflow/www/static/coverage \
+-s \
 -v \
 --logging-level=DEBUG "
 fi

scripts/ci/airflow_travis.cfg

@@ -8,6 +8,7 @@ unit_test_mode = True
 load_examples = True
 donot_pickle = False
 parallelism = 2
+fernet_key = af7CN0q6ag5U3g08IsPsw3K45U7Xa0axgVFhoh-3zB8=
 
 [webserver]
 base_url = http://localhost:8080

tests/core.py

@@ -378,6 +378,20 @@ def test_get_non_existing_var_should_not_deserialize_json_default(self):
                                              default_var=default_value,
                                              deserialize_json=True)
 
+    def test_parameterized_config_gen(self):
+
+        cfg = configuration.parameterized_config(configuration.DEFAULT_CONFIG)
+
+        # making sure some basic building blocks are present:
+        assert "[core]" in cfg
+        assert "dags_folder" in cfg
+        assert "sql_alchemy_conn" in cfg
+        assert "fernet_key" in cfg
+
+        # making sure replacement actually happened
+        assert "{AIRFLOW_HOME}" not in cfg
+        assert "{FERNET_KEY}" not in cfg
+
 
 class CliTests(unittest.TestCase):