Improve packager

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+venv/
+build/

make.py

@@ -175,7 +175,7 @@ def package():
     package_dir = os.path.join('tools', 'bin')
     package_script_path = os.path.join('tools', 'bin', 'package_application.py')
     app_path = os.path.join(g_app_name)
-    scan_for_cr(app_path)
+    # scan_for_cr(app_path)
 
     try:
         subprocess.check_output('{} {} {}'.format(g_python_cmd, package_script_path, app_path), shell=True)
@@ -371,33 +371,25 @@ def init(ceate_new_uuid):
     config.read(settings_file)
 
     # Initialize the globals based on the sdk_settings.ini contents.
-    if sdk_key in config:
-        if app_key in config[sdk_key]:
-            g_app_name = config[sdk_key][app_key]
-        else:
-            success = False
-            print('ERROR 1: The {} key does not exist in {}'.format(app_key, settings_file))
+    if sdk_key not in config:
+        print('ERROR: The {} section does not exist in {}'.format(sdk_key, settings_file))
+        return False
 
-        if ip_key in config[sdk_key]:
-            g_dev_client_ip = config[sdk_key][ip_key]
-        else:
-            success = False
-            print('ERROR 2: The {} key does not exist in {}'.format(ip_key, settings_file))
+    section = config[sdk_key]
 
-        if username_key in config[sdk_key]:
-            g_dev_client_username = config[sdk_key][username_key]
-        else:
+    for key in [app_key, ip_key, username_key, password_key]:
+        if key not in section:
+            print('ERROR: The {} key does not exist in {}'.format(key, settings_file))
+            # Maintain previous behavior of warning about each missing key by saving a flag instead of short-circuiting.
             success = False
-            print('ERROR 3: The {} key does not exist in {}'.format(username_key, settings_file))
 
-        if password_key in config[sdk_key]:
-            g_dev_client_password = config[sdk_key][password_key]
-        else:
-            success = False
-            print('ERROR 4: The {} key does not exist in {}'.format(password_key, settings_file))
-    else:
-        success = False
-        print('ERROR 5: The {} section does not exist in {}'.format(sdk_key, settings_file))
+    if not success:
+        return False
+
+    g_app_name = section[app_key]
+    g_dev_client_ip = section[ip_key]
+    g_dev_client_username = section[username_key]
+    g_dev_client_password = section[password_key]
 
     # This will also create a UUID if needed.
     get_app_uuid(ceate_new_uuid)

requirements.txt

@@ -0,0 +1,2 @@
+requests
+pyOpenSSL

tools/bin/package_application.py

@@ -18,8 +18,11 @@
 CONFIG_FILE = 'package.ini'
 SIGNATURE_FILE = 'SIGNATURE.DS'
 MANIFEST_FILE = 'MANIFEST.json'
+TEMP_FOLDER = "build/temp"
 
-BYTE_CODE_FILES = re.compile('^.*\.(pyc|pyo|pyd)$')
+SOURCE_CODE_FILE_EXTENSIONS = ('.py', '.sh')
+BYTE_CODE_FILE_EXTENSIONS = ('.pyc', '.pyo', '.pyd')
+BYTE_CODE_FILES = re.compile(r'^.*\.(pyc|pyo|pyd)$')
 BYTE_CODE_FOLDERS = re.compile('^(__pycache__)$')
 
 
@@ -33,48 +36,99 @@ def file_checksum(hash_func=hashlib.sha256, file=None):
     return h.hexdigest()
 
 
-def hash_dir(target, hash_func=hashlib.sha256):
-    hashed_files = {}
-    for path, d, f in os.walk(target):
+def get_paths_to_include(app_root):
+    """Return value: Maps relative-to-app_root to relative-to-cwd.
+    """
+    paths = {}
+    for path, d, f in os.walk(app_root):
         for fl in f:
-            if not fl.startswith('.') and not os.path.basename(path).startswith('.'):
-                # we need this be LINUX fashion!
-                if sys.platform == "win32":
-                    # swap the network\\tcp_echo to be network/tcp_echo
-                    fully_qualified_file = path.replace('\\', '/') + '/' + fl
-                else:  # else allow normal method
-                    fully_qualified_file = os.path.join(path, fl)
-                hashed_files[fully_qualified_file[len(target) + 1:]] =\
-                    file_checksum(hash_func, fully_qualified_file)
-            else:
+            if not shouldinclude2(path, fl):
+                # TODO: Get rid of this extra check.
                 print("Did not include {} in the App package.".format(fl))
+                continue
+            fully_qualified_file = os.path.join(path, fl)
+            relpath = os.path.relpath(fully_qualified_file, app_root)
+            # we need this be LINUX fashion!
+            if sys.platform == "win32":
+                # swap the network\\tcp_echo to be network/tcp_echo
+                relpath = relpath.replace('\\', '/')
+            if not shouldinclude(relpath):
+                print("Did not include {} in the App package.".format(fully_qualified_file))
+                continue
+            paths[relpath] = fully_qualified_file
+
+    return paths
+
+
+def hash_paths(paths, hash_func=hashlib.sha256):
+    hashes = {}
+    for arcpath, fpath in paths.items():
+        hashes[arcpath] = file_checksum(hash_func, fpath)
+    return hashes
+
+
+def copy_app_folder(
+        app_root,
+        out_dir=TEMP_FOLDER,
+        strip_cr=sys.platform == "win32"):
+    os.makedirs(out_dir, exist_ok=True)
+    new_paths = {}
+    for shortpath, fpath in get_paths_to_include(app_root).items():
+        outpath = os.path.join(out_dir, shortpath)
+        if strip_cr and issourcecodefile(fpath):
+            copy_without_cr(fpath, outpath)
+            shutil.copymode(fpath, outpath)  # Should this be done?
+        else:
+            shutil.copy2(fpath, outpath)  # Should we copy metadata at all?
+        new_paths[shortpath] = outpath
+    return new_paths
+
+def copy_without_cr(src, dst):
+    with open(src) as srcfile, \
+         open(dst, 'w', newline='\n') as dstfile:
+        shutil.copyfileobj(srcfile, dstfile)
+
+
+def shouldinclude2(dirpath, filename):
+    # Possible bug: A parent can have a dot and not be included, but the child's files will still be included.
+    return not filename.startswith('.') and not os.path.basename(dirpath).startswith('.')
+
 
-    return hashed_files
+def shouldinclude(fpath):
+    if fpath.lower().endswith(BYTE_CODE_FILE_EXTENSIONS):
+        return False
+    pathparts = os.path.normcase(fpath).split(os.sep)
+    if any(p.lower() == '__pycache__' for p in pathparts):
+        return False
+    if any(map(islinuxhidden, pathparts)):
+        return False
+    #... Other checks?
+    return True
+
+
+def islinuxhidden(part):
+    return part.startswith('.') and set(part) != {'.'}
+
+def issourcecodefile(fpath):
+    # Technically, it should check for programs where \r matters.
+    return fpath.lower().endswith(SOURCE_CODE_FILE_EXTENSIONS)
 
 
 def pack_package(app_root, app_name):
     print('app_root: {}'.format(app_root))
     print('app_name: {}'.format(app_name))
-    print("pack TAR:%s.tar" % app_name)
-    tar_name = "{}.tar".format(app_name)
-    tar = tarfile.open(tar_name, 'w')
-    tar.add(app_root, arcname=os.path.basename(app_root))
-    tar.close()
-
-    print("gzip archive:%s.tar.gz" % app_name)
+    # tar_name = "{}.tar".format(app_name)
+    # print("pack TAR:%s" % tar_name)
     gzip_name = "{}.tar.gz".format(app_name)
-    with open(tar_name, 'rb') as f_in:
-        with gzip.open(gzip_name, 'wb') as f_out:
-            shutil.copyfileobj(f_in, f_out)
-
-    if os.path.isfile(tar_name):
-        os.remove(tar_name)
+    print("gzip archive:%s" % gzip_name)
+    with tarfile.open(gzip_name, 'w:gz') as targz:
+        targz.add(app_root, arcname=app_name)
 
 
 def create_signature(meta_data_folder, pkey):
     manifest_file = os.path.join(meta_data_folder, MANIFEST_FILE)
+    checksum = file_checksum(hashlib.sha256, manifest_file).encode('utf-8')
     with open(os.path.join(meta_data_folder, SIGNATURE_FILE), 'wb') as sf:
-        checksum = file_checksum(hashlib.sha256, manifest_file).encode('utf-8')
         if pkey:
             sf.write(crypto.sign(pkey, checksum, 'sha256'))
         else:
@@ -94,73 +148,79 @@ def clean_manifest_folder(app_metadata_folder):
 
 def clean_bytecode_files(app_root):
     for path, dirs, files in os.walk(app_root):
-        for file in filter(lambda x: BYTE_CODE_FILES.match(x), files):
+        for file in filter(BYTE_CODE_FILES.match, files):
             os.remove(os.path.join(path, file))
-        for d in filter(lambda x: BYTE_CODE_FOLDERS.match(x), dirs):
+        for d in filter(BYTE_CODE_FOLDERS.match, dirs):
             shutil.rmtree(os.path.join(path, d))
     pass
 
 
 def package_application(app_root, pkey):
     app_root = os.path.realpath(app_root)
     app_config_file = os.path.join(app_root, CONFIG_FILE)
-    app_metadata_folder = os.path.join(app_root, META_DATA_FOLDER)
-    app_manifest_file = os.path.join(app_metadata_folder, MANIFEST_FILE)
     config = configparser.ConfigParser()
     config.read(app_config_file)
-    if not os.path.exists(app_metadata_folder):
-        os.makedirs(app_metadata_folder)
 
     for section in config.sections():
         app_name = section
         assert os.path.basename(app_root) == app_name
+        temp_root = os.path.join('build', app_name)
+        app_metadata_folder = os.path.join(temp_root, META_DATA_FOLDER)
+        app_manifest_file = os.path.join(app_metadata_folder, MANIFEST_FILE)
+        os.makedirs(app_metadata_folder, exist_ok=True)
 
         clean_manifest_folder(app_metadata_folder)
 
-        clean_bytecode_files(app_root)
-
-        pmf = {}
-        pmf['version_major'] = int(1)
-        pmf['version_minor'] = int(0)
+        # clean_bytecode_files(app_root)
+
+        pmf = {
+            'version_major': 1,
+            'version_minor': 0,
+        }
+
+        app = {
+            'name': str(section),
+            'vendor': config[section]['vendor'],
+            'notes': config[section]['notes'],
+            'version_major': int(config[section]['version_major']),
+            'version_minor': int(config[section]['version_minor']),
+            'firmware_major': int(config[section]['firmware_major']),
+            'firmware_minor': int(config[section]['firmware_minor']),
+            'restart': config[section].getboolean('restart'),
+            'reboot': config[section].getboolean('reboot'),
+            'date': datetime.datetime.now().isoformat(),
+        }
 
-        app = {}
-        app['name'] = str(section)
         try:
             app['uuid'] = config[section]['uuid']
         except KeyError:
             if not pkey:
                 app['uuid'] = str(uuid.uuid4())
             else:
                 raise
-        app['vendor'] = config[section]['vendor']
-        app['notes'] = config[section]['notes']
-        app['version_major'] = int(config[section]['version_major'])
-        app['version_minor'] = int(config[section]['version_minor'])
-        app['firmware_major'] = int(config[section]['firmware_major'])
-        app['firmware_minor'] = int(config[section]['firmware_minor'])
-        app['restart'] = config[section].getboolean('restart')
-        app['reboot'] = config[section].getboolean('reboot')
-        app['date'] = datetime.datetime.now().isoformat()
         if config.has_option(section, 'auto_start'):
             app['auto_start'] = config[section].getboolean('auto_start')
         if config.has_option(section, 'app_type'):
             app['app_type'] = int(config[section]['app_type'])
 
 
-        data = {}
-        data['pmf'] = pmf
-        data['app'] = app
+        data = {
+            'pmf': pmf,
+            'app': app,
+        }
 
-        app['files'] = hash_dir(app_root)
+        new_paths = copy_app_folder(app_root, temp_root)
+        app['files'] = hash_paths(new_paths)
 
         with open(app_manifest_file, 'w') as f:
-            f.write(json.dumps(data, indent=4, sort_keys=True))
+            json.dump(data, f, indent=4, sort_keys=True)
 
         create_signature(app_metadata_folder, pkey)
 
-        pack_package(app_root, section)
+        pack_package(temp_root, app_name)
+        #TODO: Remove temporary folder?
 
-        print('Package {}.tar.gz created'.format(section))
+        print('Package {}.tar.gz created'.format(app_name))
 
 
 def argument_list(args):