Skip to content

Commit

Permalink
Added WeightedByNumberOfVirtualPathSelector; changed baseline probabi…
Browse files Browse the repository at this point in the history
…lities
  • Loading branch information
tochilinak committed Mar 1, 2024
1 parent 7f9bfe7 commit 086702c
Show file tree
Hide file tree
Showing 10 changed files with 140 additions and 73 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import kotlin.time.Duration.Companion.seconds

class EnumerateTest: PythonTestRunnerForPrimitiveProgram(
"Enumerate",
UMachineOptions(stepLimit = 15U, timeout = 20.seconds),
UMachineOptions(stepLimit = 25U, timeout = 20.seconds),
// allowPathDiversions = true
) {
@Test
Expand Down
4 changes: 2 additions & 2 deletions usvm-python/src/test/kotlin/org/usvm/samples/ListsTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ class ListsTest : PythonTestRunnerForPrimitiveProgram(

private fun richcompareCheck(function: PyUnpinnedCallable) {
val oldOptions = options
options = UMachineOptions(stepLimit = 10U)
options = UMachineOptions(stepLimit = 15U)
allowPathDiversions = true
check2WithConcreteRun(
function,
Expand Down Expand Up @@ -262,7 +262,7 @@ class ListsTest : PythonTestRunnerForPrimitiveProgram(
@Test
fun testDoubleSubscriptAndCompare() {
val oldOptions = options
options = UMachineOptions(stepLimit = 20U, timeout = 20.seconds)
options = UMachineOptions(stepLimit = 30U, timeout = 20.seconds)
allowPathDiversions = true
check2WithConcreteRun(
constructFunction("double_subscript_and_compare", listOf(typeSystem.pythonList, typeSystem.pythonList)),
Expand Down
4 changes: 2 additions & 2 deletions usvm-python/src/test/kotlin/org/usvm/samples/MethodsTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -116,10 +116,10 @@ class MethodsTest: PythonTestRunnerForStructuredProgram("Methods", UMachineOptio
@Test
fun testPoint2Inference() {
val oldOptions = options
options = UMachineOptions(stepLimit = 2U)
options = UMachineOptions(stepLimit = 4U)
check1WithConcreteRun(
constructFunction("point2_inference", List(1) { PythonAnyType }),
eq(1),
ignoreNumberOfAnalysisResults,
standardConcolicAndConcreteChecks,
/* invariants = */ emptyList(),
/* propertiesToDiscover = */ listOf { x, _ -> x.typeName == "Point2" }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import org.usvm.runner.PythonTestRunnerForStructuredProgram
import org.usvm.test.util.checkers.eq
import org.usvm.test.util.checkers.ignoreNumberOfAnalysisResults

class SimpleCustomClassesTest: PythonTestRunnerForStructuredProgram("SimpleCustomClasses", UMachineOptions(stepLimit = 20U)) {
class SimpleCustomClassesTest: PythonTestRunnerForStructuredProgram("SimpleCustomClasses", UMachineOptions(stepLimit = 25U)) {
@Test
fun testMatmulUsage() {
check1WithConcreteRun(
Expand All @@ -25,10 +25,10 @@ class SimpleCustomClassesTest: PythonTestRunnerForStructuredProgram("SimpleCusto
@Test
fun testMatmulAndAdd() {
val oldOptions = options
options = UMachineOptions(stepLimit = 2U)
options = UMachineOptions(stepLimit = 4U)
check1WithConcreteRun(
constructFunction("matmul_and_add", List(1) { PythonAnyType }),
eq(1),
ignoreNumberOfAnalysisResults,
compareConcolicAndConcreteTypes,
/* invariants = */ emptyList(),
/* propertiesToDiscover = */ listOf { x, res ->
Expand All @@ -41,7 +41,7 @@ class SimpleCustomClassesTest: PythonTestRunnerForStructuredProgram("SimpleCusto
@Test
fun testMatmulAddAndSub() {
val oldOptions = options
options = UMachineOptions(stepLimit = 6U)
options = UMachineOptions(stepLimit = 10U)
check1WithConcreteRun(
constructFunction("matmul_add_and_sub", List(1) { PythonAnyType }),
ignoreNumberOfAnalysisResults,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ class SimpleTypeInferenceTest: PythonTestRunnerForPrimitiveProgram("SimpleTypeIn
fun testAddAndCompare() {
val oldOptions = options
allowPathDiversions = true
options = UMachineOptions(stepLimit = 100U)
options = UMachineOptions(stepLimit = 120U)
timeoutPerRunMs = 2000
check2WithConcreteRun(
constructFunction("add_and_compare", List(2) { PythonAnyType }),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package org.usvm.python.ps

enum class PyPathSelectorType {
BaselinePriorityDfs,
BaselineWeightedDfs,
BaselinePriorityRandomTree,
BaselinePriorityWeightedByNumberOfVirtual,
BaselinePriorityPlusTypeRatingByHintsDfs,
DelayedForkByInstructionPriorityDfs,
DelayedForkByInstructionWeightedDfs,
DelayedForkByInstructionWeightedRandomTree
BaselinePriorityDfs, // passes tests
BaselineWeightedDfs, // passes tests
BaselinePriorityNumberOfVirtualDfs, // passes tests
BaselinePriorityPlusTypeRatingByHintsDfs, // passes tests
DelayedForkByInstructionPriorityDfs, // fails tests
DelayedForkByInstructionWeightedDfs, // passes tests
DelayedForkByInstructionWeightedRandomTree, // passes tests
DelayedForkByInstructionPriorityNumberOfVirtualDfs, // passes tests
DelayedForkByInstructionWeightedNumberOfVirtualDfs, // fails test MultiplyAndCompare
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import kotlin.random.Random
class PyMachine(
private val program: PyProgram,
private val typeSystem: PythonTypeSystem,
private val pathSelectorType: PyPathSelectorType = PyPathSelectorType.DelayedForkByInstructionWeightedRandomTree,
private val pathSelectorType: PyPathSelectorType = PyPathSelectorType.BaselinePriorityDfs,
private val printErrorMsg: Boolean = false
): UMachine<PyState>() {
private val ctx = PyContext(typeSystem)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,24 +1,14 @@
package org.usvm.machine.ps

import mu.KLogging
import org.usvm.algorithms.RandomizedPriorityCollection
import org.usvm.machine.PyContext
import org.usvm.machine.PyState
import org.usvm.machine.model.PyModelHolder
import org.usvm.machine.ps.strategies.impls.*
import org.usvm.machine.results.observers.NewStateObserver
import org.usvm.machine.symbolicobjects.interpretSymbolicPythonObject
import org.usvm.machine.symbolicobjects.rendering.PyObjectModelBuilder
import org.usvm.ps.DfsPathSelector
import org.usvm.ps.RandomTreePathSelector
import org.usvm.ps.WeightedPathSelector
import org.usvm.python.model.PyTupleObject
import org.usvm.python.model.calculateNumberOfMocks
import org.usvm.python.ps.PyPathSelectorType
import kotlin.math.max
import kotlin.random.Random

private val logger = object : KLogging() {}.logger

fun createPyPathSelector(
initialState: PyState,
Expand All @@ -34,11 +24,8 @@ fun createPyPathSelector(
PyPathSelectorType.BaselineWeightedDfs ->
createBaselineWeightedDfsPyPathSelector(ctx, random, newStateObserver)

PyPathSelectorType.BaselinePriorityRandomTree ->
createBaselinePriorityRandomTreePyPathSelector(initialState, ctx, random, newStateObserver)

PyPathSelectorType.BaselinePriorityWeightedByNumberOfVirtual ->
createBaselineWeightedByNumberOfVirtualPyPathSelector(ctx, random, newStateObserver)
PyPathSelectorType.BaselinePriorityNumberOfVirtualDfs ->
createBaselinePriorityNumberOfVirtualDfsPyPathSelector(ctx, random, newStateObserver)

PyPathSelectorType.BaselinePriorityPlusTypeRatingByHintsDfs ->
createTypeRatingByHintsDfsPyPathSelector(ctx, random, newStateObserver)
Expand All @@ -51,6 +38,12 @@ fun createPyPathSelector(

PyPathSelectorType.DelayedForkByInstructionWeightedRandomTree ->
createDelayedForkByInstructionWeightedRandomTreePyPathSelector(initialState, ctx, random, newStateObserver)

PyPathSelectorType.DelayedForkByInstructionPriorityNumberOfVirtualDfs ->
createDelayedForkByInstructionPriorityNumberOfVirtualDfsPyPathSelector(ctx, random, newStateObserver)

PyPathSelectorType.DelayedForkByInstructionWeightedNumberOfVirtualDfs ->
createDelayedForkByInstructionWeightedNumberOfVirtualDfsPyPathSelector(ctx, random, newStateObserver)
}
selector.add(listOf(initialState))
return selector
Expand All @@ -69,25 +62,6 @@ fun createBaselinePriorityDfsPyPathSelector(
newStateObserver
)

fun createBaselinePriorityRandomTreePyPathSelector(
initialState: PyState,
ctx: PyContext,
random: Random,
newStateObserver: NewStateObserver
): PyVirtualPathSelector<*, *> =
PyVirtualPathSelector(
ctx,
makeBaselinePriorityActionStrategy(random),
BaselineDelayedForkStrategy(),
BaselineDFGraphCreation {
RandomTreePathSelector.fromRoot(
initialState.pathNode,
randomNonNegativeInt = { random.nextInt(0, it) }
)
},
newStateObserver
)

fun createBaselineWeightedDfsPyPathSelector(
ctx: PyContext,
random: Random,
Expand All @@ -101,7 +75,7 @@ fun createBaselineWeightedDfsPyPathSelector(
newStateObserver
)

fun createBaselineWeightedByNumberOfVirtualPyPathSelector(
fun createBaselinePriorityNumberOfVirtualDfsPyPathSelector(
ctx: PyContext,
random: Random,
newStateObserver: NewStateObserver
Expand All @@ -111,23 +85,9 @@ fun createBaselineWeightedByNumberOfVirtualPyPathSelector(
makeBaselinePriorityActionStrategy(random),
BaselineDelayedForkStrategy(),
BaselineDFGraphCreation {
WeightedPathSelector(
priorityCollectionFactory = {
RandomizedPriorityCollection(compareBy { it.id }) { random.nextDouble() }
},
weighter = {
val modelHolder = PyModelHolder(it.pyModel)
val builder = PyObjectModelBuilder(it, modelHolder)
val models = it.inputSymbols.map { symbol ->
val interpreted = interpretSymbolicPythonObject(modelHolder, it.memory, symbol)
builder.convert(interpreted)
}
val tupleOfModels = PyTupleObject(models)
val mocks = calculateNumberOfMocks(tupleOfModels)
logger.debug { "Mocks of state $it: $mocks" }
1.0 / max(1, 10 * mocks)
}
)
WeightedByNumberOfVirtualPathSelector(random) {
DfsPathSelector()
}
},
newStateObserver
)
Expand Down Expand Up @@ -177,6 +137,40 @@ fun createDelayedForkByInstructionWeightedRandomTreePyPathSelector(
newStateObserver
)

fun createDelayedForkByInstructionPriorityNumberOfVirtualDfsPyPathSelector(
ctx: PyContext,
random: Random,
newStateObserver: NewStateObserver
): PyVirtualPathSelector<*, *> =
PyVirtualPathSelector(
ctx,
makeDelayedForkByInstructionPriorityStrategy(random),
BaselineDelayedForkStrategy(),
DelayedForkByInstructionGraphCreation {
WeightedByNumberOfVirtualPathSelector(random) {
DfsPathSelector()
}
},
newStateObserver
)

fun createDelayedForkByInstructionWeightedNumberOfVirtualDfsPyPathSelector(
ctx: PyContext,
random: Random,
newStateObserver: NewStateObserver
): PyVirtualPathSelector<*, *> =
PyVirtualPathSelector(
ctx,
makeDelayedForkByInstructionWeightedStrategy(random),
BaselineDelayedForkStrategy(),
DelayedForkByInstructionGraphCreation {
WeightedByNumberOfVirtualPathSelector(random) {
DfsPathSelector()
}
},
newStateObserver
)

fun createTypeRatingByHintsDfsPyPathSelector(
ctx: PyContext,
random: Random,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package org.usvm.machine.ps

import org.usvm.UPathSelector
import org.usvm.machine.PyState
import org.usvm.machine.model.PyModelHolder
import org.usvm.machine.symbolicobjects.interpretSymbolicPythonObject
import org.usvm.machine.symbolicobjects.rendering.PyObjectModelBuilder
import org.usvm.python.model.PyTupleObject
import org.usvm.python.model.calculateNumberOfMocks
import kotlin.math.max
import kotlin.random.Random


class WeightedByNumberOfVirtualPathSelector(
private val random: Random,
private val baseBaseSelectorCreation: () -> UPathSelector<PyState>
): UPathSelector<PyState> {
private val selectors = mutableMapOf<Int, UPathSelector<PyState>>()
private val selectorOfState = mutableMapOf<PyState, UPathSelector<PyState>?>()

override fun isEmpty(): Boolean {
return selectors.all { it.value.isEmpty() }
}

override fun peek(): PyState {
val availableNumbers = selectors.mapNotNull { (number, selector) ->
if (selector.isEmpty()) null else number
}
val chosenNumber = weightedRandom(random, availableNumbers) { mocks -> 1.0 / max(1, 10 * mocks) }
val selector = selectors[chosenNumber]!!
return selector.peek()
}

override fun update(state: PyState) {
remove(state)
add(state)
}

override fun add(states: Collection<PyState>) {
states.forEach { add(it) }
}

private fun add(state: PyState) {
val numberOfVirtual = calculateNumberOfVirtual(state)
if (selectors[numberOfVirtual] == null)
selectors[numberOfVirtual] = baseBaseSelectorCreation()
val selector = selectors[numberOfVirtual]!!
selector.add(listOf(state))
selectorOfState[state] = selector
}

override fun remove(state: PyState) {
val selector = selectorOfState[state] ?: error("State was not in path selector")
selector.remove(state)
selectorOfState[state] = null
}

private fun calculateNumberOfVirtual(state: PyState): Int {
val modelHolder = PyModelHolder(state.pyModel)
val builder = PyObjectModelBuilder(state, modelHolder)
val models = state.inputSymbols.map { symbol ->
val interpreted = interpretSymbolicPythonObject(modelHolder, state.memory, symbol)
builder.convert(interpreted)
}
val tupleOfModels = PyTupleObject(models)
return calculateNumberOfMocks(tupleOfModels)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,12 @@ import org.usvm.machine.PyState
import org.usvm.machine.ps.strategies.*
import kotlin.random.Random

val baselineProbabilities = listOf(1.0, 0.6, 0.9, 0.7, 1.0)
val baselineWeights = listOf(100.0, 0.6, 0.35, 0.04, 0.01)
val baselineProbabilities = listOf(1.0, 0.6, 0.875, 0.8, 1.0)
private val probNegations = baselineProbabilities
.drop(1)
.runningFold(1.0) { acc, p -> acc * (1 - p) }
val baselineWeights = //listOf(100.0, 0.6, 0.35, 0.04, 0.01)
listOf(100.0) + (baselineProbabilities.drop(1) zip probNegations.dropLast(1)).map { (a, b) -> a * b }

fun makeBaselinePriorityActionStrategy(
random: Random
Expand Down

0 comments on commit 086702c

Please sign in to comment.