Make rule writing and debugging easier

config/config.json5

@@ -11,12 +11,15 @@
   //specifies the rule's parent directory, default is ./config/rules
   //  "rulePath": "config/rules",
 
-  //print more info about this rule
-  //  "debugRule": "unZipSlip",
+  //print more info about rules, specify "all" for all rules
+  //  "debugRule": "unZipSlip.json",
 
   //output log level
   //  "logLevel": 0,
 
+  //output jimple code for debug
+  //  "jimpleSource": true,
+
   //display the decompiled java code in the results
   //  "javaSource": true,
 }

src/main/kotlin/net/bytedance/security/app/AnalyzeStepByStep.kt

@@ -36,14 +36,17 @@ import kotlin.io.path.pathString
 import kotlin.streams.toList
 
 class AnalyzeStepByStep {
-    suspend fun loadRules(ruleList: String, targetSdk: Int): Rules {
-        val rulePathList = if (ruleList.isNotEmpty())
-            ruleList.split(",").map { "${getConfig().rulePath}/${it.trim()}" }.toList()
-        else
-            withContext(Dispatchers.IO) {
-                Files.walk(Paths.get(getConfig().rulePath), 1)
-            }.filter { it.pathString.endsWith(".json") }.map { it.pathString }
-                .toList()
+    suspend fun loadRules(targetSdk: Int): Rules {
+        val config = getConfig()
+        if (config.rules.isEmpty()) {
+            config.rules = withContext(Dispatchers.IO) {
+                Files.walk(Paths.get(config.rulePath), 1) }
+                .filter { it.pathString.endsWith(".json") || it.pathString.endsWith(".json5")}
+                .map { it.fileName }.toList().joinToString(separator = ",")
+        }
+        val rulePathList = config.rules.split(",")
+            .map { "${config.rulePath}/${it.trim()}" }.toList()
+
         val rules = Rules(rulePathList, RuleFactory())
         rules.loadRules(targetSdk)
         return rules
@@ -103,10 +106,11 @@ class AnalyzeStepByStep {
         // reduce time
         val excludeList = ArrayList<String>()
         excludeList.add("java.*")
+        excludeList.add("javax.*")
         excludeList.add("org.*")
         excludeList.add("sun.*")
-        //        excludeList.add("android.*");
-//        excludeList.add("androidx.*");
+        // excludeList.add("android.*")
+        // excludeList.add("androidx.*")
         Options.v().set_exclude(excludeList)
         // do not load body in exclude list
         Options.v().set_no_bodies_for_excluded(true)
@@ -166,7 +170,7 @@ class AnalyzeStepByStep {
         Options.v().set_debug(false)
         Options.v().set_verbose(false)
         Options.v().set_validate(false)
-//        Options.v().set_keep_line_number(true)
+        // Options.v().set_keep_line_number(true)
         setExclude()
         logInfo("loadNecessaryClasses")
         try {

src/main/kotlin/net/bytedance/security/app/ArgumentConfig.kt

@@ -31,6 +31,7 @@ class ArgumentConfig(
     //max pointer analysis time in second for each entry point
     var maxPointerAnalyzeTime: Int = 600,
     var javaSource: Boolean? = false,
+    var jimpleSource: Boolean? = false,
     /**
      * If you have OOM problems, try lowering this value, such as 1, to save memory
      */
@@ -82,6 +83,7 @@ class ArgumentConfig(
                 outPath = "$wd/out",
                 rules = "",
                 javaSource = false,
+                jimpleSource = false,
                 maxPointerAnalyzeTime = 600,
                 maxThread = 4,
                 manifestTrace = 3,

src/main/kotlin/net/bytedance/security/app/StaticAnalyzeMain.kt

@@ -44,7 +44,7 @@ object StaticAnalyzeMain {
             AnalyzeStepByStep.TYPE.APK,
             apkPath,
             "${argumentConfig.configPath}/tools/platforms",
-            argumentConfig.outPath
+            "${argumentConfig.outPath}/soot"
         )
         logInfo("soot init done")
         PLUtils.createCustomClass()
@@ -54,7 +54,7 @@ object StaticAnalyzeMain {
         profiler.parseApk.end()
 
         profiler.preProcessor.start()
-        val rules = v3.loadRules(argumentConfig.rules, AndroidUtils.TargetSdk)
+        val rules = v3.loadRules(AndroidUtils.TargetSdk)
         logInfo("rules loaded")
         val ctx = v3.createContext(rules)
         profiler.preProcessor.end()

src/main/kotlin/net/bytedance/security/app/android/AndroidUtils.kt

@@ -34,6 +34,7 @@ import soot.RefType
 import soot.Scene
 import soot.SootClass
 import soot.SootMethod
+import soot.PackManager
 import soot.jimple.Constant
 import soot.jimple.InstanceInvokeExpr
 import soot.jimple.Stmt
@@ -54,6 +55,7 @@ import java.nio.charset.StandardCharsets
 import java.util.concurrent.TimeUnit
 import java.util.zip.ZipEntry
 import java.util.zip.ZipFile
+import java.lang.reflect.Method
 import kotlin.system.exitProcess
 
 
@@ -252,12 +254,27 @@ object AndroidUtils {
                 e.printStackTrace()
             }
         }
-        apkAbsPath = apkPath
+
         if (getConfig().javaSource == true) {
             Log.logDebug("Dex to java code")
             dexToJava(apkPath, outPath, jadxPath)
         }
+        if (getConfig().jimpleSource == true) {
+            Log.logDebug("Dex to jimple code")
+            val excludeList = arrayListOf(
+                "android.", "androidx.", "kotlin.", "kotlinx.", "java.", "javax.",
+                "dalvik.", "org.", "sun.", "com.google.")
+
+            val writeList = Scene.v().applicationClasses.filter { sootClass ->
+                excludeList.none { exclude -> sootClass.name.startsWith(exclude) }
+            }
 
+            val method = PackManager.v()::class.java.getDeclaredMethod("writeOutput", Iterator::class.java)
+            method.isAccessible = true
+            method.invoke(PackManager.v(), writeList.iterator())
+        }
+
+        apkAbsPath = apkPath
         val targetAPK = File(apkAbsPath!!)
         Log.logDebug("Load resource")
         resources = ARSCFileParser()

src/main/kotlin/net/bytedance/security/app/engineconfig/IgnoreListsConfig.kt

@@ -28,30 +28,17 @@ class IgnoreListsConfig(ignoreListData: IgnoreListsData) {
     private var methodSigSet: HashSet<String> = HashSet()
 
     init {
-        ignoreListData.PackageName?.forEach {
-            packageNameSet.add(it)
-        }
-        ignoreListData.MethodName?.forEach {
-            methodNameSet.add(it)
-        }
+        ignoreListData.PackageName?.forEach { packageNameSet.add(it) }
+        ignoreListData.MethodName?.forEach { methodNameSet.add(it) }
         ignoreListData.MethodSignature?.forEach { methodSigSet.add(it) }
-
     }
 
     fun isInIgnoreList(className: String, methodName: String, methodSig: String): Boolean {
         return containsPackageName(className) || containsMethodName(methodName) || containsMethodSig(methodSig)
     }
 
     private fun containsPackageName(className: String): Boolean {
-        if (packageNameSet.contains(className)) {
-            return true
-        }
-        for (ignorePackageName in packageNameSet) {
-            if (className.startsWith(ignorePackageName)) {
-                return true
-            }
-        }
-        return false
+        return packageNameSet.any { className.startsWith(it) }
     }
 
     private fun containsMethodName(methodName: String): Boolean {
@@ -61,5 +48,4 @@ class IgnoreListsConfig(ignoreListData: IgnoreListsData) {
     private fun containsMethodSig(methodSig: String): Boolean {
         return methodSigSet.contains(methodSig)
     }
-
-}
+}

src/main/kotlin/net/bytedance/security/app/engineconfig/LibraryConfig.kt

@@ -20,26 +20,12 @@ package net.bytedance.security.app.engineconfig
 class LibraryConfig(val libraryData: LibraryData) {
 
     private fun isInExcludeLibrary(className: String): Boolean {
-        for (excludeContain in libraryData.ExcludeLibraryContains) {
-            if (className.contains(excludeContain)) {
-                return true
-            }
-        }
-        return false
+        return libraryData.ExcludeLibraryContains.any { className.contains(it) }
     }
 
     fun isLibraryClass(className: String): Boolean {
-        for (packageName in libraryData.Package) {
-            //If it's library. It is necessary to continue to check
-            // whether it belongs to the whitelist which needs to be analyzed
-            if (className.startsWith(packageName)) {
-                // Only those not on the whitelist are considered libraries
-                if (!isInExcludeLibrary(className)) {
-                    return true
-                }
-            }
-        }
-        return false
+        // those belong to Package and not belong to ExcludeLibraryContains
+        return libraryData.Package.any { className.startsWith(it) && !isInExcludeLibrary(className) }
     }
 
     fun isLibraryMethod(methodSig: String): Boolean {
@@ -50,5 +36,4 @@ class LibraryConfig(val libraryData: LibraryData) {
     fun setPackage(packages: List<String>) {
         libraryData.Package = packages
     }
-
-}
+}

src/main/kotlin/net/bytedance/security/app/pathfinder/TaintPathFinder.kt

@@ -41,6 +41,8 @@ import soot.jimple.IntConstant
 import soot.jimple.LongConstant
 import soot.jimple.NumericConstant
 import soot.jimple.StringConstant
+import java.nio.file.Files
+import java.nio.file.Paths
 import java.util.*
 
 /**
@@ -149,10 +151,7 @@ class TaintPathFinder(
         if (constNumberModeRule.targetNumberArr == null) {
             return true
         }
-        if (constNumberModeRule.targetNumberArr.contains(constNumber.toInt())) {
-            return true
-        }
-        return false
+        return constNumberModeRule.targetNumberArr.contains(constNumber.toInt())
     }
 
     /**
@@ -264,32 +263,35 @@ class TaintPathFinder(
     ) {
         if (isThisSolverNeedLog()) {
             val sb = StringBuilder()
+            val outPath = "${getConfig().outPath}/log/${this.rule.name}"
+            Files.createDirectories(Paths.get(outPath))
+
             val sinkTaintedSet = HashSet<PLPointer>()
             for (sink in sinkPtrSet) {
                 sinkTaintedSet.addAll(analyzeContext.collectReversePropagation(sink, rule.primTypeAsTaint))
             }
             sb.append("sinkPtrSet=${sinkPtrSet.toSortedSet()}, taint sinkNodeSet: ${sinkTaintedSet.toSortedSet()}\n")
-            PLUtils.writeFile(getConfig().outPath + "/sink.log", sb.toString())
+            PLUtils.writeFile("$outPath/sink.log", sb.toString())
             sb.clear()
-            sb.append("\n\n\n\n\n\nsrcPtr=${srcPtr}, taint sourceNodeSet:\n")
+
+            sb.append("srcPtr=${srcPtr}, taint sourceNodeSet:\n")
             val srcTaintedSet = analyzeContext.collectPropagation(srcPtr, rule.primTypeAsTaint)
             sb.append("\n\nsrcTaintedSet=${srcTaintedSet.toSortedSet()}")
-            PLUtils.writeFile(getConfig().outPath + "/source.log", sb.toString())
+            PLUtils.writeFile("$outPath/source.log", sb.toString())
             sb.clear()
+
+            PLUtils.writeFile(
+                "$outPath/ptrToSet.log",
+                analyzeContext.pointerToObjectSet.toSortedMap().toFormatedString())
             PLUtils.writeFile(
-                getConfig().outPath + "/ptrToSet.log",
-                analyzeContext.pointerToObjectSet.toSortedMap().toFormatedString()
-            )
+                "$outPath/taintPtrFlowGraph.log",
+                analyzeContext.variableFlowGraph.toSortedMap().toFormatedString())
             PLUtils.writeFile(
-                getConfig().outPath + "/taintPtrFlowGraph.log",
-                analyzeContext.variableFlowGraph.toSortedMap().toFormatedString()
-            )
+                "$outPath/ptrFlowGraph.log",
+                analyzeContext.pointerFlowGraph.toSortedMap().toFormatedString())
             PLUtils.writeFile(
-                getConfig().outPath + "/ptrFlowGraph.log",
-                analyzeContext.pointerFlowGraph.toSortedMap().toFormatedString()
-            )
-            PLUtils.writeFile(getConfig().outPath + "rm.log", analyzeContext.rm.toSortedSet().toFormatedString())
-//            exitProcess(3)
+                "$outPath/rm.log",
+                analyzeContext.rm.toSortedSet().toFormatedString())
         }
         val g = analyzeContext.variableFlowGraph
         val path = bfsSearch(srcPtr, sinkPtrSet, g, getConfig().maxPathLength, rule.name) ?: return

src/main/kotlin/net/bytedance/security/app/rules/TaintFlowRule.kt

@@ -45,7 +45,16 @@ abstract class TaintFlowRule(name: String, ruleData: RuleData) : AbstractRule(na
     }
 
     fun isThisRuleNeedLog(): Boolean {
-        return getConfig().debugRule == this.name
+        val config = getConfig()
+        val ruleNameList = let {
+            fun String.process() = this.split(",").map { it.trim().substringBeforeLast('.') }
+            when (config.debugRule) {
+                "" -> emptyList()
+                "all" -> config.rules.process()
+                else -> config.debugRule.process()
+            }
+        }
+        return ruleNameList.contains(this.name)
     }
 
     fun isThroughEnable(): Boolean {

src/main/kotlin/net/bytedance/security/app/util/helper.kt

@@ -61,6 +61,7 @@ fun newFunctionSignature(methodSig: String): FunctionSignature {
     for (i in 1 until methodSig.length - 1) {
         when (val c = methodSig[i]) {
             ':' -> if (state != MethodSignatureParseState.Class) {
+                logErr("Format Error $methodSig")
                 exitProcess(-2)
             } else {
                 // state = ParseState.Space
@@ -80,7 +81,10 @@ fun newFunctionSignature(methodSig: String): FunctionSignature {
                         state = MethodSignatureParseState.FunctionName
                     }
 
-                    else -> exitProcess(-7)
+                    else -> {
+                        logErr("Format Error $methodSig")
+                        exitProcess(-7)
+                    }
                 }
             }
 
@@ -91,7 +95,10 @@ fun newFunctionSignature(methodSig: String): FunctionSignature {
                         s = ""
                     }
 
-                    else -> exitProcess(-8)
+                    else -> {
+                        logErr("Format Error $methodSig")
+                        exitProcess(-8)
+                    }
                 }
             }
 
@@ -103,7 +110,10 @@ fun newFunctionSignature(methodSig: String): FunctionSignature {
                         state = MethodSignatureParseState.Argument
                     }
 
-                    else -> exitProcess(-9)
+                    else -> {
+                        logErr("Format Error $methodSig")
+                        exitProcess(-9)
+                    }
                 }
             }
 
@@ -114,7 +124,10 @@ fun newFunctionSignature(methodSig: String): FunctionSignature {
                         s = ""
                     }
 
-                    else -> exitProcess(-10)
+                    else -> {
+                        logErr("Format Error $methodSig")
+                        exitProcess(-10)
+                    }
                 }
             }