Add Lua VM memory to memory overhead, now that it's part of zmalloc

To complement the work done in redis#13133
contains some refactoring, and also a new field in MEMORY STATS

additionally, clear scripts and stats between tests in external mode

src/eval.c

@@ -730,15 +730,15 @@ NULL
     }
 }
 
-unsigned long evalMemory(void) {
+unsigned long evalScriptsMemoryVM(void) {
     return luaMemory(lctx.lua);
 }
 
 dict* evalScriptsDict(void) {
     return lctx.lua_scripts;
 }
 
-unsigned long evalScriptsMemory(void) {
+unsigned long evalScriptsMemoryEngine(void) {
     return lctx.lua_scripts_mem +
             dictMemUsage(lctx.lua_scripts) +
             dictSize(lctx.lua_scripts) * sizeof(luaScript) +

src/functions.c

@@ -1063,7 +1063,7 @@ void functionLoadCommand(client *c) {
 }
 
 /* Return memory usage of all the engines combine */
-unsigned long functionsMemory(void) {
+unsigned long functionsMemoryVM(void) {
     dictIterator *iter = dictGetIterator(engines);
     dictEntry *entry = NULL;
     size_t engines_memory = 0;
@@ -1078,7 +1078,7 @@ unsigned long functionsMemory(void) {
 }
 
 /* Return memory overhead of all the engines combine */
-unsigned long functionsMemoryOverhead(void) {
+unsigned long functionsMemoryEngine(void) {
     size_t memory_overhead = dictMemUsage(engines);
     memory_overhead += dictMemUsage(curr_functions_lib_ctx->functions);
     memory_overhead += sizeof(functionsLibCtx);

src/functions.h

@@ -102,8 +102,8 @@ struct functionLibInfo {
 
 int functionsRegisterEngine(const char *engine_name, engine *engine_ctx);
 sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx, size_t timeout);
-unsigned long functionsMemory(void);
-unsigned long functionsMemoryOverhead(void);
+unsigned long functionsMemoryVM(void);
+unsigned long functionsMemoryEngine(void);
 unsigned long functionsNum(void);
 unsigned long functionsLibNum(void);
 dict* functionsLibGet(void);

src/object.c

@@ -1230,12 +1230,16 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
     mh->aof_buffer = mem;
     mem_total+=mem;
 
-    mem = evalScriptsMemory();
-    mh->lua_caches = mem;
+    mem = evalScriptsMemoryEngine();
+    mh->eval_caches = mem;
     mem_total+=mem;
-    mh->functions_caches = functionsMemoryOverhead();
+    mh->functions_caches = functionsMemoryEngine();
     mem_total+=mh->functions_caches;
 
+    mh->script_vm = evalScriptsMemoryVM();
+    mh->script_vm += functionsMemoryVM();
+    mem_total+=mh->script_vm;
+
     for (j = 0; j < server.dbnum; j++) {
         redisDb *db = server.db+j;
         if (!kvstoreNumAllocatedDicts(db->keys)) continue;
@@ -1583,11 +1587,14 @@ NULL
         addReplyLongLong(c,mh->aof_buffer);
 
         addReplyBulkCString(c,"lua.caches");
-        addReplyLongLong(c,mh->lua_caches);
+        addReplyLongLong(c,mh->eval_caches);
 
         addReplyBulkCString(c,"functions.caches");
         addReplyLongLong(c,mh->functions_caches);
 
+        addReplyBulkCString(c,"script.VMs");
+        addReplyLongLong(c,mh->script_vm);
+
         for (size_t j = 0; j < mh->num_dbs; j++) {
             char dbname[32];
             snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid);

src/server.c

@@ -5700,8 +5700,8 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
         size_t zmalloc_used = zmalloc_used_memory();
         size_t total_system_mem = server.system_memory_size;
         const char *evict_policy = evictPolicyToString();
-        long long memory_lua = evalMemory();
-        long long memory_functions = functionsMemory();
+        long long memory_lua = evalScriptsMemoryVM();
+        long long memory_functions = functionsMemoryVM();
         struct redisMemOverhead *mh = getMemoryOverheadData();
 
         /* Peak memory is updated from time to time by serverCron() so it
@@ -5716,7 +5716,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
         bytesToHuman(total_system_hmem,sizeof(total_system_hmem),total_system_mem);
         bytesToHuman(used_memory_lua_hmem,sizeof(used_memory_lua_hmem),memory_lua);
         bytesToHuman(used_memory_vm_total_hmem,sizeof(used_memory_vm_total_hmem),memory_functions + memory_lua);
-        bytesToHuman(used_memory_scripts_hmem,sizeof(used_memory_scripts_hmem),mh->lua_caches + mh->functions_caches);
+        bytesToHuman(used_memory_scripts_hmem,sizeof(used_memory_scripts_hmem),mh->eval_caches + mh->functions_caches);
         bytesToHuman(used_memory_rss_hmem,sizeof(used_memory_rss_hmem),server.cron_malloc_stats.process_rss);
         bytesToHuman(maxmemory_hmem,sizeof(maxmemory_hmem),server.maxmemory);
 
@@ -5742,15 +5742,15 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
             "used_memory_lua:%lld\r\n", memory_lua, /* deprecated, renamed to used_memory_vm_eval */
             "used_memory_vm_eval:%lld\r\n", memory_lua,
             "used_memory_lua_human:%s\r\n", used_memory_lua_hmem, /* deprecated */
-            "used_memory_scripts_eval:%lld\r\n", (long long)mh->lua_caches,
+            "used_memory_scripts_eval:%lld\r\n", (long long)mh->eval_caches,
             "number_of_cached_scripts:%lu\r\n", dictSize(evalScriptsDict()),
             "number_of_functions:%lu\r\n", functionsNum(),
             "number_of_libraries:%lu\r\n", functionsLibNum(),
             "used_memory_vm_functions:%lld\r\n", memory_functions,
             "used_memory_vm_total:%lld\r\n", memory_functions + memory_lua,
             "used_memory_vm_total_human:%s\r\n", used_memory_vm_total_hmem,
             "used_memory_functions:%lld\r\n", (long long)mh->functions_caches,
-            "used_memory_scripts:%lld\r\n", (long long)mh->lua_caches + (long long)mh->functions_caches,
+            "used_memory_scripts:%lld\r\n", (long long)mh->eval_caches + (long long)mh->functions_caches,
             "used_memory_scripts_human:%s\r\n", used_memory_scripts_hmem,
             "maxmemory:%lld\r\n", server.maxmemory,
             "maxmemory_human:%s\r\n", maxmemory_hmem,

src/server.h

@@ -1401,8 +1401,9 @@ struct redisMemOverhead {
     size_t clients_normal;
     size_t cluster_links;
     size_t aof_buffer;
-    size_t lua_caches;
+    size_t eval_caches;
     size_t functions_caches;
+    size_t script_vm;
     size_t overhead_total;
     size_t dataset;
     size_t total_keys;
@@ -3510,9 +3511,9 @@ int ldbIsEnabled(void);
 void ldbLog(sds entry);
 void ldbLogRedisReply(char *reply);
 void sha1hex(char *digest, char *script, size_t len);
-unsigned long evalMemory(void);
+unsigned long evalScriptsMemoryVM(void);
 dict* evalScriptsDict(void);
-unsigned long evalScriptsMemory(void);
+unsigned long evalScriptsMemoryEngine(void);
 uint64_t evalGetCommandFlags(client *c, uint64_t orig_flags);
 uint64_t fcallGetCommandFlags(client *c, uint64_t orig_flags);
 int isInsideYieldingLongCommand(void);

tests/support/server.tcl

@@ -373,6 +373,8 @@ proc run_external_server_test {code overrides} {
 
     r flushall
     r function flush
+    r script flush
+    r config resetstat
 
     # store configs
     set saved_config {}