Merge pull request #396 from qilingframework/dev

Getting Ready for 1.1

ChangeLog

@@ -1,5 +1,14 @@
 This file details the changelog of Qiling Framework.
 
+
+[Version 1.1]: July 24th, 2020
+
+- More refactors and bug fixes
+- Adding DLL images for PE coverage tracing
+- Add hook_mem_invalid
+- More UEFI API
+
+
 ------------------------------------
 [Version 1.1-rc1]: July 17th, 2020
 

TODO

@@ -48,12 +48,12 @@ Android
 -------------------- CHECKLIST before TAG --------------------
 
 1. core.py
-    - __version__ = "1.1" + "-XXXXX"
+    - __version__ = "1.1" + "-rc1"
 2. setup.py
     - 'Development Status :: 3 - Alpha'
 3. ChangeLog    
 4. commit and push
 5. Merge dev into master
-6. git tag 1.1-Alpha2
+6. git tag 1.1-rc1
 7. git push origin --tags
 8. check for new Pypi package

examples/fuzzing/fuzz_x8664_linux.py

@@ -19,7 +19,7 @@
 import sys, os
 from binascii import hexlify
 
-from capstone.x86_const import *
+from capstone import *
 
 sys.path.append("../..")
 from qiling import *

examples/hello_linuxx8664_intercept.py

@@ -5,6 +5,7 @@
 import sys
 sys.path.append("..")
 from qiling import *
+from qiling.const import *
 
 def write_onenter(ql, arg1, arg2, arg3, *args):
     print("enter write syscall!")
@@ -18,6 +19,6 @@ def write_onexit(ql, arg1, arg2, arg3, *args):
 if __name__ == "__main__":
     ql = Qiling(["rootfs/x8664_linux/bin/x8664_hello"], "rootfs/x8664_linux", output="debug")
     # ql.set_api('puts', my_puts)
-    ql.set_syscall(1, write_onenter)
-    ql.set_syscall(1, write_onexit)
+    ql.set_syscall(1, write_onenter, QL_INTERCEPT.ENTER)
+    ql.set_syscall(1, write_onexit, QL_INTERCEPT.EXIT)
     ql.run()

qiling/core.py

@@ -18,7 +18,7 @@
 from .core_utils import QLCoreUtils
 from .extensions.debugger import ql_debugger_init
 
-__version__ = "1.1" + "-rc1"
+__version__ = "1.1"
 
 class Qiling(QLCoreStructs, QLCoreHooks, QLCoreUtils):    
     def __init__(

qiling/core_hooks.py

@@ -326,6 +326,9 @@ def hook_mem_fetch_invalid(self, callback, user_data=None, begin=1, end=0):
 
     def hook_mem_valid(self, callback, user_data=None, begin=1, end=0):
         return self.ql_hook(UC_HOOK_MEM_VALID, callback, user_data, begin, end)
+
+    def hook_mem_invalid(self, callback, user_data=None, begin=1, end=0):
+        return self.ql_hook(UC_HOOK_MEM_INVALID, callback, user_data, begin, end)
 
 
     # a convenient API to set callback for a single address

qiling/extensions/debugger/gdbserver/gdbserver.py

@@ -172,22 +172,23 @@ def handle_g(subcmd):
                 elif self.ql.archtype == QL_ARCH.ARM:
                     mode = self.ql.arch.check_thumb()
 
-                    for reg in self.tables[QL_ARCH.ARM]:
+                    for reg in self.tables[QL_ARCH.ARM][:16]:
                         r = self.ql.reg.read(reg)
                         if mode == UC_MODE_THUMB and reg == "pc":
                             r += 1
-
+                        elif mode != UC_MODE_THUMB and reg == "pc":
+                            r += 4
                         tmp = self.ql.arch.addr_to_str(r)
                         s += tmp
 
-                elif self.ql.archtype== QL_ARCH.ARM64:
-                    for reg in self.tables[QL_ARCH.ARM64]:
+                elif self.ql.archtype == QL_ARCH.ARM64:
+                    for reg in self.tables[QL_ARCH.ARM64][:33]:
                         r = self.ql.reg.read(reg)
                         tmp = self.ql.arch.addr_to_str(r)
                         s += tmp
 
-                elif self.ql.archtype== QL_ARCH.MIPS:
-                    for reg in self.tables[QL_ARCH.MIPS]:
+                elif self.ql.archtype == QL_ARCH.MIPS:
+                    for reg in self.tables[QL_ARCH.MIPS][:38]:
                         r = self.ql.reg.read(reg)
                         if self.ql.archendian == QL_ENDIAN.EB:
                             tmp = self.ql.arch.addr_to_str(r, endian ="little")
@@ -200,14 +201,14 @@ def handle_g(subcmd):
 
             def handle_G(subcmd):
                 count = 0
-                if self.ql.archtype== QL_ARCH.X86:
+                if self.ql.archtype == QL_ARCH.X86:
                     for i in range(0, len(subcmd), 8):
                         reg_data = subcmd[i:i+7]
                         reg_data = int(reg_data, 16)
                         self.ql.reg.write(self.tables[QL_ARCH.X86][count], reg_data)
                         count += 1
 
-                elif self.ql.archtype== QL_ARCH.X8664:
+                elif self.ql.archtype == QL_ARCH.X8664:
                     for i in range(0, 17*16, 16):
                         reg_data = subcmd[i:i+15]
                         reg_data = int(reg_data, 16)
@@ -219,21 +220,21 @@ def handle_G(subcmd):
                         self.ql.reg.write(self.tables[QL_ARCH.X8664][count], reg_data)
                         count += 1
 
-                elif self.ql.archtype== QL_ARCH.ARM:
+                elif self.ql.archtype == QL_ARCH.ARM:
                     for i in range(0, len(subcmd), 8):
                         reg_data = subcmd[i:i + 7]
                         reg_data = int(reg_data, 16)
                         self.ql.reg.write(self.tables[QL_ARCH.ARM][count], reg_data)
                         count += 1
 
-                elif self.ql.archtype== QL_ARCH.ARM64:
+                elif self.ql.archtype == QL_ARCH.ARM64:
                     for i in range(0, len(subcmd), 16):
                         reg_data = subcmd[i:i+15]
                         reg_data = int(reg_data, 16)
                         self.ql.reg.write(self.tables[QL_ARCH.ARM64][count], reg_data)
                         count += 1
 
-                elif self.ql.archtype== QL_ARCH.MIPS:
+                elif self.ql.archtype == QL_ARCH.MIPS:
                     for i in range(0, len(subcmd), 8):
                         reg_data = subcmd[i:i+7]
                         reg_data = int(reg_data, 16)
@@ -664,7 +665,7 @@ def handle_v(subcmd):
                         subcmd = subcmd[1].split(':')
                         if subcmd[0] in ('c', 'C05'):
                             handle_c(subcmd)
-                        elif subcmd[0] in ('S', 's'):
+                        elif subcmd[0] in ('S', 's', 'S05'):
                             handle_s(subcmd)
                 else:
                     self.send("")

qiling/loader/elf.py

@@ -371,7 +371,7 @@ def run(self):
         self.argv = self.ql.argv
         self.ql.mem.map(stack_address, stack_size, info="[stack]") 
         self.load_with_ld(stack_address + stack_size, argv = self.argv, env = self.env)
-        self.stack_address  = int(self.new_stack)
+        self.stack_address  = self.new_stack
         self.ql.reg.arch_sp = self.stack_address
 
         if self.ql.ostype == QL_OS.FREEBSD:
@@ -466,6 +466,7 @@ def load_with_ld(self, stack_addr, load_address = -1, argv = [], env = {}):
 
         loaded_mem_end = load_address + mem_end
         if loaded_mem_end > _mem_e:
+
             self.ql.mem.map(_mem_e, loaded_mem_end-_mem_e, info=self.path)
             self.ql.dprint(D_INFO, "[+] load 0x%x - 0x%x" % (_mem_e, loaded_mem_end)) # make sure we map all PT_LOAD tagged area
 
@@ -560,7 +561,7 @@ def load_with_ld(self, stack_addr, load_address = -1, argv = [], env = {}):
 
         # Set AUX
 
-        # self.ql.mem.write(int(new_stack) - 4, self.ql.pack32(0x11111111))
+        # self.ql.mem.write(new_stack - 4, self.ql.pack32(0x11111111))
         # new_stack = new_stack - 4
         # rand_addr = new_stack - 4
 
@@ -599,7 +600,7 @@ def load_with_ld(self, stack_addr, load_address = -1, argv = [], env = {}):
         elf_table += self.NEW_AUX_ENT(AT_NULL, 0)
         elf_table += b'\x00' * (0x10 - (new_stack - len(elf_table)) & 0xf)
 
-        self.ql.mem.write(int(new_stack - len(elf_table)), elf_table)
+        self.ql.mem.write(new_stack - len(elf_table), elf_table)
         new_stack = new_stack - len(elf_table)
 
         # self.ql.reg.write(UC_X86_REG_RDI, new_stack + 8)

qiling/loader/pe.py

@@ -96,6 +96,9 @@ def load_dll(self, dll_name):
         # add dll to ldr data
         self.add_ldr_data_table_entry(dll_name)
 
+        # add DLL to coverage images
+        self.images.append(self.coverage_image(dll_base, dll_base+dll_len, path))
+
         self.ql.nprint("[+] Done with loading %s" % path)
 
         return dll_base

qiling/loader/pe_uefi.py

@@ -20,6 +20,8 @@
 from qiling.os.uefi.runtime import *
 from qiling.os.uefi.dxe_service import *
 from qiling.os.uefi.smm_base2_protocol import *
+from qiling.os.uefi.mm_access_protocol import *
+from qiling.os.uefi.smm_sw_dispatch2_protocol import *
 
 from qiling.os.windows.fncc import *
 
@@ -191,18 +193,19 @@ def run(self):
         system_table = EFI_SYSTEM_TABLE()
         system_table_heap_ptr = system_table_heap + ctypes.sizeof(EFI_SYSTEM_TABLE)
 
-        runtime_services_ptr = system_table_heap_ptr
-        system_table.RuntimeServices = runtime_services_ptr
+        self.runtime_services_ptr = system_table_heap_ptr
+        system_table.RuntimeServices = self.runtime_services_ptr
         system_table_heap_ptr += ctypes.sizeof(EFI_RUNTIME_SERVICES)
-        system_table_heap_ptr, runtime_services = hook_EFI_RUNTIME_SERVICES(self.ql, system_table_heap_ptr)
+        system_table_heap_ptr, self.runtime_services = hook_EFI_RUNTIME_SERVICES(self.ql, system_table_heap_ptr)
 
         boot_services_ptr = system_table_heap_ptr
         system_table.BootServices = boot_services_ptr
         system_table_heap_ptr += ctypes.sizeof(EFI_BOOT_SERVICES)
-        system_table_heap_ptr, boot_services = hook_EFI_BOOT_SERVICES(self.ql, system_table_heap_ptr)
+        system_table_heap_ptr, boot_services, efi_mm_system_table = hook_EFI_BOOT_SERVICES(self.ql, system_table_heap_ptr)
 
         self.efi_configuration_table_ptr = system_table_heap_ptr
         system_table.ConfigurationTable = self.efi_configuration_table_ptr
+        efi_mm_system_table.MmConfigurationTable = self.efi_configuration_table_ptr
         system_table.NumberOfTableEntries = 2
         system_table_heap_ptr += ctypes.sizeof(EFI_CONFIGURATION_TABLE) * 100 # We don't expect more then a few entries.
         efi_configuration_table = EFI_CONFIGURATION_TABLE()
@@ -225,11 +228,23 @@ def run(self):
         self.efi_configuration_table = [self.ql.os.profile["HOB_LIST"]["guid"]]
         self.ql.mem.write(self.efi_configuration_table_ptr, convert_struct_to_bytes(efi_configuration_table))
 
+        self.mm_system_table_ptr = system_table_heap_ptr
+        system_table_heap_ptr += ctypes.sizeof(EFI_MM_SYSTEM_TABLE)
         self.smm_base2_protocol_ptr = system_table_heap_ptr
         system_table_heap_ptr += ctypes.sizeof(EFI_SMM_BASE2_PROTOCOL)
-        system_table_heap_ptr, smm_base2_protocol = install_EFI_SMM_BASE2_PROTOCOL(self.ql, system_table_heap_ptr)
+        system_table_heap_ptr, smm_base2_protocol, efi_mm_system_table = install_EFI_SMM_BASE2_PROTOCOL(self.ql, system_table_heap_ptr, efi_mm_system_table)
         self.handle_dict[1] = {self.ql.os.profile.get("EFI_SMM_BASE2_PROTOCOL", "guid"): self.smm_base2_protocol_ptr}
 
+        self.mm_access_protocol_ptr = system_table_heap_ptr
+        system_table_heap_ptr += ctypes.sizeof(EFI_MM_ACCESS_PROTOCOL)
+        system_table_heap_ptr, mm_access_protocol = install_EFI_MM_ACCESS_PROTOCOL(self.ql, system_table_heap_ptr)
+        self.handle_dict[1][self.ql.os.profile.get("EFI_MM_ACCESS_PROTOCOL", "guid")] = self.mm_access_protocol_ptr
+
+        self.smm_sw_dispatch2_protocol_ptr = system_table_heap_ptr
+        system_table_heap_ptr += ctypes.sizeof(EFI_SMM_SW_DISPATCH2_PROTOCOL)
+        system_table_heap_ptr, smm_sw_dispatch2_protocol = install_EFI_SMM_SW_DISPATCH2_PROTOCOL(self.ql, system_table_heap_ptr)
+        self.handle_dict[1][self.ql.os.profile.get("EFI_SMM_SW_DISPATCH2_PROTOCOL", "guid")] = self.smm_sw_dispatch2_protocol_ptr
+
         self.dxe_services_ptr = system_table_heap_ptr
         system_table_heap_ptr += ctypes.sizeof(EFI_DXE_SERVICES)
         system_table_heap_ptr, dxe_services = install_EFI_DXE_SERVICES(self.ql, system_table_heap_ptr)
@@ -249,10 +264,13 @@ def run(self):
         self.efi_configuration_table.append(self.ql.os.profile.get("DXE_SERVICE_TABLE", "guid"))
 
 
-        self.ql.mem.write(runtime_services_ptr, convert_struct_to_bytes(runtime_services))
+        self.ql.mem.write(self.runtime_services_ptr, convert_struct_to_bytes(self.runtime_services))
         self.ql.mem.write(boot_services_ptr, convert_struct_to_bytes(boot_services))
         self.ql.mem.write(self.system_table_ptr, convert_struct_to_bytes(system_table))
+        self.ql.mem.write(self.mm_system_table_ptr, convert_struct_to_bytes(efi_mm_system_table))
         self.ql.mem.write(self.smm_base2_protocol_ptr, convert_struct_to_bytes(smm_base2_protocol))
+        self.ql.mem.write(self.mm_access_protocol_ptr, convert_struct_to_bytes(mm_access_protocol))
+        self.ql.mem.write(self.smm_sw_dispatch2_protocol_ptr, convert_struct_to_bytes(smm_sw_dispatch2_protocol))
         self.ql.mem.write(self.dxe_services_ptr, convert_struct_to_bytes(dxe_services))
 
         for dependency in self.ql.argv:
@@ -272,3 +290,7 @@ def run(self):
         self.OOO_EOE_callbacks = []
 
         self.execute_next_module()
+
+    def restore_runtime_services(self):
+        self.ql.mem.write(self.runtime_services_ptr, convert_struct_to_bytes(self.runtime_services))
+

qiling/os/filestruct.py

@@ -16,6 +16,7 @@ class ql_file:
     def __init__(self, path, fd):
         self.__path = path
         self.__fd = fd
+        self._is_map_shared = False
 
     @classmethod
     def open(self, open_path, open_flags, open_mode):

qiling/os/linux/function_hook.py

@@ -734,7 +734,7 @@ def __init__(self, ql, phoff, phnum, phentsize, load_base, hook_mem):
         if self.ql.archtype == QL_ARCH.MIPS and self.plt_got != None and self.mips_gotsym != None and self.mips_local_gotno != None and self.mips_symtabno != None:
             self.show_dynsym_name(self.mips_gotsym, self.mips_symtabno)
 
-        self.ql.mem.map(hook_mem, 0x2000, perms=7, info="hook mem")
+        self.ql.mem.map(hook_mem, 0x2000, perms=7, info="[hook_mem]")
         self.ql.mem.write(hook_mem, ins * (0x1000 // len(ins)))
 
         self.free_list = [_ for _ in range(0, 0x1000, 0x10)]

qiling/os/linux/linux.py

@@ -111,13 +111,14 @@ def run(self):
 
                     thread_management.run()
                 else:
-                    if self.ql.loader.elf_entry != self.ql.loader.entry_point:
-                        self.ql.emu_start(self.ql.loader.entry_point, self.ql.loader.elf_entry, self.ql.timeout)
-                        self.ql.enable_lib_patch()
-                        self.run_function_after_load()
 
                     if  self.ql.entry_point is not None:
                         self.ql.loader.elf_entry = self.ql.entry_point
+
+                    elif self.ql.loader.elf_entry != self.ql.loader.entry_point:
+                        self.ql.emu_start(self.ql.loader.entry_point, self.ql.loader.elf_entry, self.ql.timeout)
+                        self.ql.enable_lib_patch()
+                        self.run_function_after_load()
 
                     self.ql.emu_start(self.ql.loader.elf_entry, self.exit_point, self.ql.timeout, self.ql.count)
 

qiling/os/memory.py

@@ -166,12 +166,14 @@ def restore(self, mem_dict):
             perm = value[2]
             info = value[3]
             mem_read = bytes(value[4])
-
-            if self.is_mapped(start, start-end) == False:
+
+            self.ql.dprint(4,"restore key: %i 0x%x 0x%x %s" % (key, start, end, info))
+            if self.is_mapped(start, end-start) == False:
+                self.ql.dprint(4,"mapping 0x%x 0x%x mapsize 0x%x" % (start, end, end-start))
                 self.map(start, end-start, perms=perm, info=info)
 
+            self.ql.dprint(4,"writing 0x%x size 0x%x write_size 0x%x " % (start, end-start, len(mem_read)))
             self.write(start, mem_read)
-
 
     def read(self, addr: int, size: int) -> bytearray:
         return self.ql.uc.mem_read(addr, size)
@@ -380,8 +382,8 @@ def map(self, addr, size, perms=UC_PROT_ALL, info=None, ptr=None):
         '''
         if ptr == None:
             if self.is_mapped(addr, size) == False:
-               self.ql.uc.mem_map(addr, size, perms)
-               self.add_mapinfo(addr, addr + size, perms, info if info else "[mapped]")
+                self.ql.uc.mem_map(addr, size, perms)
+                self.add_mapinfo(addr, addr + size, perms, info if info else "[mapped]")
             else:
                 raise QlMemoryMappedError("[!] Memory Mapped")    
         else: