dev(hansbug): add new tests and features

.gitignore

@@ -769,7 +769,6 @@ pip-selfcheck.json
 *.log
 *.debug
 *.str
-*.zip
 *.tmp
 *.os
 *.js
@@ -873,7 +872,6 @@ luac.out
 # luarocks build files
 *.src.rock
 *.zip
-*.tar.gz
 
 # Object files
 *.o

Makefile

@@ -67,3 +67,5 @@ testfile:
 		rm -rf $(abspath ${TESTFILE_DIR}/tar_template-simple.tar) && \
 		tar -cvf $(abspath ${TESTFILE_DIR}/tar_template-simple.tar) * && \
 		cd ../..
+	cp "$(abspath ${TESTFILE_DIR}/gztar_template-simple.tar.gz)" "$(abspath ${TEMPLATES_DIR}/test/template/raw.tar.gz)"
+	cp "$(abspath ${TESTFILE_DIR}/gztar_template-simple.tar.gz)" "$(abspath ${TEMPLATES_DIR}/test/template/.unpacked.tar.gz)"

igm/render/archive.py

@@ -0,0 +1,17 @@
+import os
+import shutil
+from typing import Optional, Mapping, Any
+
+from .base import RenderJob
+from ..utils import get_archive_type
+
+
+class ArchiveUnpackJob(RenderJob):
+    def __init__(self, srcpath: str, dstpath: str, extras: Optional[Mapping[str, Any]] = None):
+        RenderJob.__init__(self, srcpath, dstpath)
+        _ = extras
+
+    def _run(self):
+        archive_fmt = get_archive_type(self.srcpath)
+        os.makedirs(self.dstpath, exist_ok=True)
+        shutil.unpack_archive(self.srcpath, self.dstpath, archive_fmt)

igm/render/base.py

@@ -25,9 +25,9 @@ def run(self, silent: bool = False):
 
 
 class DirectoryBasedTask(RenderTask):
-    def __init__(self, srcdir: str, dststr: str, extras: Optional[Mapping[str, Any]] = None):
+    def __init__(self, srcdir: str, dstdir: str, extras: Optional[Mapping[str, Any]] = None):
         self.srcdir = srcdir
-        self.dstdir = dststr
+        self.dstdir = dstdir
         self._extras = dict(extras or {})
         RenderTask.__init__(self, list(self._yield_jobs()))
 
@@ -43,6 +43,7 @@ def run(self, silent: bool = False):
             jobs = self
             pgbar = None
 
+        os.makedirs(self.dstdir, exist_ok=True)
         # run jobs
         for job in jobs:
             if pgbar:

igm/render/template.py

@@ -4,26 +4,55 @@
 from functools import partial
 from typing import List, Dict, Any, Optional, Mapping
 
+from hbutils.system import copy
 from jinja2 import Environment
 from potc import transobj as _potc_transobj
 from potc.fixture.imports import ImportStatement
 
+from .archive import ArchiveUnpackJob
 from .base import RenderJob, DirectoryBasedTask
 from .imports import PyImport
+from .script import ScriptJob
+from ..utils import get_archive_type, is_binary_file, splitext
 
 
 class NotTemplateFile(Exception):
     pass
 
 
 class IGMRenderTask(DirectoryBasedTask):
-    def __init__(self, srcdir: str, dststr: str, extras: Optional[Mapping[str, Any]] = None):
-        DirectoryBasedTask.__init__(self, srcdir, dststr, extras)
+    def __init__(self, srcdir: str, dstdir: str, extras: Optional[Mapping[str, Any]] = None):
+        DirectoryBasedTask.__init__(self, srcdir, dstdir, extras)
 
     def _load_job_by_file(self, relfile: str):
-        srcfile = os.path.join(self.srcdir, relfile)
-        dstfile = os.path.join(self.dstdir, relfile)
-        return TemplateJob(srcfile, dstfile, self._extras)
+        directory, filename = os.path.split(os.path.normcase(relfile))
+        if filename.startswith('.') and filename.endswith('.py'):  # script file or template
+            if filename.startswith('..'):  # ..xxx.py --> .xxx.py (template)
+                return get_common_job(
+                    os.path.join(self.srcdir, relfile),
+                    os.path.join(self.dstdir, directory, filename[1:]),
+                    self._extras
+                )
+            else:  # .xxx.py --> xxx (script)
+                body, _ = splitext(filename)
+                return ScriptJob(
+                    os.path.join(self.srcdir, relfile),
+                    os.path.join(self.dstdir, directory, body[1:]),
+                    self._extras
+                )
+        elif filename.startswith('.') and get_archive_type(filename):  # unpack archive file
+            body, _ = splitext(filename)
+            return ArchiveUnpackJob(  # .xxx.zip --> xxx (unzip)
+                os.path.join(self.srcdir, relfile),
+                os.path.join(self.dstdir, directory, body[1:]),
+                self._extras
+            )
+        else:  # common cases
+            return get_common_job(  # xxx.yy --> xxx.yy (template/binary copy)
+                os.path.join(self.srcdir, relfile),
+                os.path.join(self.dstdir, relfile),
+                self._extras
+            )
 
     def _yield_jobs(self):
         for curdir, subdirs, files in os.walk(self.srcdir):
@@ -32,10 +61,17 @@ def _yield_jobs(self):
                 curfile = os.path.join(cur_reldir, file)
                 try:
                     yield self._load_job_by_file(curfile)
-                except NotTemplateFile:
+                except NotTemplateFile:  # pragma: no cover
                     pass
 
 
+def get_common_job(src, dst, extras):
+    if is_binary_file(src):
+        return CopyJob(src, dst, extras)
+    else:
+        return TemplateJob(src, dst, extras)
+
+
 class TemplateImportWarning(Warning):
     pass
 
@@ -105,3 +141,16 @@ def _run(self):
                 f'These import statement is suggested to added in template {self.srcpath!r}:{os.linesep}'
                 f'{os.linesep.join(unimports)}'
             ))
+
+
+class CopyJob(RenderJob):
+    def __init__(self, srcpath: str, dstpath: str, extras: Optional[Mapping[str, Any]] = None):
+        RenderJob.__init__(self, srcpath, dstpath)
+        _ = extras
+
+    def _run(self):
+        dstdir, _ = os.path.split(self.dstpath)
+        if dstdir:
+            os.makedirs(dstdir, exist_ok=True)
+
+        copy(self.srcpath, self.dstpath)

igm/utils/__init__.py

@@ -1,4 +1,5 @@
 from .archive import unpack_archive, get_archive_type
+from .file import is_binary_file, get_file_ext, splitext
 from .globals import get_global_env, get_globals
 from .path import normpath
 from .pythonpath import with_pythonpath

igm/utils/file.py

@@ -0,0 +1,58 @@
+import mimetypes
+import os
+from itertools import chain
+
+_TEXT_CHARS = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
+
+
+def _is_binary_string(data: bytes) -> bool:
+    return bool(data.translate(None, _TEXT_CHARS))
+
+
+def _take_sample(filename, size=1024) -> bytes:
+    with open(filename, 'rb') as f:
+        return f.read(size)
+
+
+def is_binary_file(filename) -> bool:
+    """
+    Overview:
+        Check if the given file is binary file.
+
+    :param filename: Filename.
+    :return: Is binary file or not.
+    """
+    return _is_binary_string(_take_sample(filename))
+
+
+def _iter_extensions():
+    for n1, n2 in chain(*map(
+            lambda x: x.items(),
+            [mimetypes.types_map, mimetypes.common_types, mimetypes.encodings_map, mimetypes.suffix_map]
+    )):
+        if n1.startswith('.'):
+            yield n1
+        if n2.startswith('.'):
+            yield n2
+
+
+def get_file_ext(filename) -> str:
+    filename = os.path.normcase(filename)
+
+    ext = ''
+    for exist_ext in _iter_extensions():
+        if filename.endswith(exist_ext) and len(exist_ext) > len(ext):
+            ext = exist_ext
+
+    if not ext:
+        _, ext = os.path.splitext(filename)
+
+    return ext
+
+
+def splitext(filename):
+    ext = get_file_ext(filename)
+    if ext:
+        return filename[:-len(ext)], ext
+    else:
+        return filename, ''

igm/utils/url.py

@@ -1,9 +1,10 @@
 import mimetypes
 import os.path
-from itertools import chain
 from typing import Optional
 from urllib.parse import urlparse, unquote
 
+from .file import get_file_ext, splitext
+
 
 def get_url_filename(url: str, content_type: Optional[str] = None) -> str:
     """
@@ -16,7 +17,7 @@ def get_url_filename(url: str, content_type: Optional[str] = None) -> str:
     """
     url_parsed = urlparse(url)
     filename = os.path.basename(unquote(url_parsed.path))
-    _, ext = os.path.splitext(filename)
+    _, ext = splitext(filename)
     if content_type and not ext:
         actual_ext = mimetypes.guess_extension(content_type)
         if actual_ext and not os.path.normcase(filename).endswith(actual_ext):
@@ -25,17 +26,6 @@ def get_url_filename(url: str, content_type: Optional[str] = None) -> str:
     return filename
 
 
-def _iter_extensions():
-    for n1, n2 in chain(*map(
-            lambda x: x.items(),
-            [mimetypes.types_map, mimetypes.common_types, mimetypes.encodings_map, mimetypes.suffix_map]
-    )):
-        if n1.startswith('.'):
-            yield n1
-        if n2.startswith('.'):
-            yield n2
-
-
 def get_url_ext(url: str, content_type: Optional[str] = None) -> str:
     """
     Overview:
@@ -46,13 +36,4 @@ def get_url_ext(url: str, content_type: Optional[str] = None) -> str:
     :return: File extension, including ``.tar.gz``.
     """
     filename = get_url_filename(url, content_type)
-    filename = os.path.normcase(filename)
-    ext = ''
-    for exist_ext in _iter_extensions():
-        if filename.endswith(exist_ext) and len(exist_ext) > len(ext):
-            ext = exist_ext
-
-    if not ext:
-        _, ext = os.path.splitext(filename)
-
-    return ext
+    return get_file_ext(filename)

templates/test/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 igm4ai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

templates/test/README.md

@@ -0,0 +1,3 @@
+# template-test
+
+Just a test template for IGM's unittest.

templates/test/inquirer.py

@@ -0,0 +1,44 @@
+from InquirerPy import inquirer
+
+from igm.conf import InquireRestart
+from igm.env import env
+
+_LAST_NAME = ""
+_LAST_AGE = 18
+_LAST_GENDER = "Male"
+
+
+def inquire_func():
+    global _LAST_AGE, _LAST_NAME, _LAST_GENDER
+
+    name = env.NAME or inquirer.text(message="What's your name:", default=_LAST_NAME).execute()
+    age = int(env.AGE or inquirer.number(
+        message="What's your age:",
+        min_allowed=1,
+        float_allowed=False,
+        invalid_message='Age should not be less than 1.',
+        default=_LAST_AGE,
+    ).execute())
+    gender = str(env.GENDER or inquirer.select(
+        message="Your gender?",
+        choices=["Male", "Female", "Others"],
+        default=_LAST_GENDER,
+    ).execute())
+
+    if env.NON_CONFIRM:
+        confirm = True
+    else:
+        confirm = inquirer.confirm(message=f"{name}, {age}, {gender}, confirm?").execute()
+
+    if confirm:
+        return {
+            'name': name,
+            'age': age,
+            'gender': gender
+        }
+    else:
+        # save this time's fillings
+        _LAST_NAME = name
+        _LAST_AGE = age
+        _LAST_GENDER = gender
+        raise InquireRestart('Not confirmed.')

templates/test/meta.py

@@ -0,0 +1,9 @@
+from igm.conf import igm_setup
+from inquirer import inquire_func
+
+igm_setup(
+    name='test',
+    version='0.0.1',
+    description='Just a test template for IGM.',
+    inquire=inquire_func,
+)

templates/test/requirements.txt

@@ -0,0 +1 @@
+inquirerpy>=0.3.3

templates/test/template/..main.py

@@ -0,0 +1,14 @@
+cpus = {{ sys.cpu.num }}
+mem_size = {{ sys.memory.total | str | potc }}
+os = {{ sys.os.type | str | potc }}
+python = {{ sys.python | str | potc }}
+{% if sys.cuda %}
+cuda_version = {{ sys.cuda.version | str | potc }}
+gpu_num = {{ sys.gpu.num | potc }}
+{% endif %}
+
+print('This is your first try!')
+print(f'UR running {python} on {os}, with a {cpus} core {mem_size} device.')
+{% if sys.cuda %}
+print(f'CUDA {cuda_version} is also detected, with {gpu_num} gpu(s).')
+{% endif %}

templates/test/template/README.md

@@ -0,0 +1,9 @@
+# hello world for {{ user.name }}
+
+This is a hello world project of igm created by {{ user.name | potc }} (age: `{{ user.age }}`).
+
+You can start this project by the following command:
+
+```python
+python main.py
+```

templates/test/template/main.py

@@ -0,0 +1,14 @@
+cpus = {{ sys.cpu.num }}
+mem_size = {{ sys.memory.total | str | potc }}
+os = {{ sys.os.type | str | potc }}
+python = {{ sys.python | str | potc }}
+{% if sys.cuda %}
+cuda_version = {{ sys.cuda.version | str | potc }}
+gpu_num = {{ sys.gpu.num | potc }}
+{% endif %}
+
+print('This is your first try!')
+print(f'UR running {python} on {os}, with a {cpus} core {mem_size} device.')
+{% if sys.cuda %}
+print(f'CUDA {cuda_version} is also detected, with {gpu_num} gpu(s).')
+{% endif %}