chore: basic isEnabled benchmarks

.gitignore

@@ -28,3 +28,5 @@ devenv.local.nix
 
 # Direnv
 .direnv*
+
+**/BenchmarkDotNet.Artifacts/

dotnet-engine/README.md

@@ -19,3 +19,11 @@ dotnet build
 ```bash
 dotnet test
 ```
+
+## Running the benchmarks
+
+```bash
+dotnet run --project Yggdrasil.Benchmarks -c Release
+```
+
+Output can be read in Yggdrasil.Benchmarks/BenchmarkDotNet.Artifacts/results

dotnet-engine/Yggdrasil.Benchmarks/Program.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Text.Json;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Running;
+using Newtonsoft.Json.Linq;
+using Yggdrasil;
+
+[Config(typeof(Config))]
+public class YggBench
+{
+    private YggdrasilEngine yggdrasilEngine;
+
+    private class Config : ManualConfig
+    {
+        public Config()
+        {
+            AddColumn(BenchmarkDotNet.Columns.StatisticColumn.OperationsPerSecond);
+        }
+    }
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        var basePath = Path.Combine(
+            "..",
+            "..",
+            "..",
+            "..",
+            "..",
+            "..",
+            "..",
+            "..",
+            "..",
+            "client-specification",
+            "specifications"
+        );
+        var suitePath = Path.Combine(basePath, "01-simple-examples.json");
+        var suiteData = JObject.Parse(File.ReadAllText(suitePath));
+
+        yggdrasilEngine = new YggdrasilEngine();
+        yggdrasilEngine.TakeState(suiteData["state"].ToString());
+    }
+
+    [Benchmark]
+    public void IsFeatureAEnabled()
+    {
+        yggdrasilEngine.IsEnabled("Feature.A", new Context());
+    }
+}
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        var summary = BenchmarkRunner.Run<YggBench>();
+    }
+}

dotnet-engine/Yggdrasil.Benchmarks/Yggdrasil.Benchmarks.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.10" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Yggdrasil.Engine\Yggdrasil.Engine.csproj" />
+  </ItemGroup>
+
+</Project>

dotnet-engine/dotnet-engine.sln

@@ -7,6 +7,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Yggdrasil.Engine", "Yggdras
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Yggdrasil.Engine.Tests", "Yggdrasil.Engine.Tests\Yggdrasil.Engine.Tests.csproj", "{53D34278-E160-4E54-857B-AF74F665B8BB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Yggdrasil.Benchmarks", "Yggdrasil.Benchmarks\Yggdrasil.Benchmarks.csproj", "{8203D4B8-44F5-4A0E-8AEE-C82BBF0F9CF9}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -24,5 +26,9 @@ Global
 		{53D34278-E160-4E54-857B-AF74F665B8BB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{53D34278-E160-4E54-857B-AF74F665B8BB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{53D34278-E160-4E54-857B-AF74F665B8BB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8203D4B8-44F5-4A0E-8AEE-C82BBF0F9CF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8203D4B8-44F5-4A0E-8AEE-C82BBF0F9CF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8203D4B8-44F5-4A0E-8AEE-C82BBF0F9CF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8203D4B8-44F5-4A0E-8AEE-C82BBF0F9CF9}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

java-engine/README.md

@@ -20,4 +20,15 @@ Then tests can be run with:
 
 ```bash
 gradle test
-```
+```
+
+You can run the benchmarks with:
+
+```bash
+gradle jmh
+```
+
+Or if gradle has a cached run:
+```bash
+gradle jmh --rerun-tasks
+```

java-engine/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     // Apply the java-library plugin for API and implementation separation.
     id 'java-library'
+    id "me.champeau.jmh" version "0.7.2"
 }
 
 repositories {
@@ -22,6 +23,10 @@ dependencies {
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.1'
 
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2'
+
+    jmh 'org.openjdk.jmh:jmh-core:1.33'
+
+    jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.33'
 }
 
 // Apply a specific Java toolchain to ease working on different environments.
@@ -35,3 +40,14 @@ tasks.named('test') {
     // Use JUnit Platform for unit tests.
     useJUnitPlatform()
 }
+
+jmh {
+    version = '1.33'
+    fork = 2
+    includeTests = false
+    iterations = 5
+    timeOnIteration = '5s'
+    warmup = '5s'
+    warmupForks = 1
+    warmupIterations = 3
+}

java-engine/settings.gradle

@@ -6,6 +6,12 @@
  * Detailed information about configuring a multi-project build in Gradle can be found
  * in the user manual at https://docs.gradle.org/8.1/userguide/multi_project_builds.html
  */
+pluginManagement {
+    repositories {
+        gradlePluginPortal()
+        mavenCentral()
+    }
+}
 
 plugins {
     // Apply the foojay-resolver plugin to allow automatic download of JDKs

java-engine/src/jmh/java/UnleashEngineBenchmark.java

@@ -0,0 +1,50 @@
+package io.getunleash.engine;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Scope;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.nio.file.Paths;
+import java.io.IOException;
+
+
+@State(Scope.Thread)
+public class UnleashEngineBenchmark {
+
+    private UnleashEngine engine;
+    private final String featureFilePath = "../client-specification/specifications/01-simple-examples.json";
+
+    private static String loadFeaturesFromFile(String filePath) throws IOException {
+        ObjectMapper mapper = new ObjectMapper();
+        JsonNode jsonNode = mapper.readTree(Paths.get(filePath).toFile());
+        JsonNode state = jsonNode.get("state");
+        return state.toString();
+    }
+
+    @Setup
+    public void setUp() {
+        engine = new UnleashEngine(new YggdrasilFFI("../target/release"));
+        try {
+            engine.takeState(loadFeaturesFromFile(featureFilePath));
+        } catch (Exception e) {
+            System.out.println("Failed to setup benchmarks");
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    @Benchmark
+    public void benchmarkFeatureToggle() {
+        Context context = new Context();
+        try {
+            Boolean result = engine.isEnabled("Feature.A", context);
+        } catch (Exception e) {
+            System.out.println("Exception caught during benchmark, this is no longer a valid benchmark so early exiting");
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+}

java-engine/src/main/java/io/getunleash/engine/UnleashEngine.java

@@ -77,7 +77,6 @@ private <T> T read(Pointer pointer, TypeReference<T> typeReference) {
         String str = pointer.getString(0, UTF_8);
         yggdrasil.freeResponse(pointer);
         try {
-            System.out.println(str); // TODO use a logging library. SLF4J?
             return reader.forType(typeReference).readValue(str);
         } catch (IOException e) {
             throw new YggdrasilParseException(str, typeReference.getClass(), e);

ruby-engine/Gemfile

@@ -8,4 +8,5 @@ gem "ffi"
 gem "fiddle"
 group :development do
   gem "get_process_mem", "~> 0.2.7"
+  gem "benchmark-ips", "~> 2.12.0"
 end

ruby-engine/README.md

@@ -15,10 +15,18 @@ Then you can run the tests with:
 rspec
 ```
 
+Benchmarks can be run with:
+
+```bash
+rspec scripts/benchmark.rb
+```
+And should produce human readable output.
+
+
 There's also a `mem_check.rb` in the scripts folder. This is not a bullet proof test, but it can be helpful for detecting large leaks. This requires human interaction - you need to read the output and understand what it's telling you, so it's not run as part of the test suite.
 
 ```bash
-ruby scripts/mem_check.rb
+rspec scripts/mem_check.rb
 ```
 
 ## Build

ruby-engine/scripts/benchmark.rb

@@ -0,0 +1,11 @@
+require 'benchmark/ips'
+require_relative '../lib/unleash_engine'
+
+unleash_engine = UnleashEngine.new
+suite_path = File.join('../client-specification/specifications', '01-simple-examples.json')
+suite_data = JSON.parse(File.read(suite_path))
+json_client_features = suite_data['state'].to_json
+
+Benchmark.ips do |x|
+  x.report("enabled") { unleash_engine.enabled?('Feature.A', {}) }
+end