feat: use strtok from c (fix #608) (#612)

* feat: use strtok from c

* forgotten strtok in test folder

.editorconfig

@@ -31,3 +31,7 @@ indent_size = 3
 [src/lua/zencode.lua]
 indent_style = tab
 indent_size = 4
+
+[src/lua/zencode_data.lua]
+indent_style = space
+indent_size = 2

src/lua/zencode_array.lua

@@ -391,7 +391,7 @@ When("create the array by splitting '' at ''", function(data_name, sep_name)
         local sep = uscore(have(sep_name):octet():string())
         ZEN.assert(#sep == 1, "You can only split with respect to one character")
         empty'array'
-        local strings = strtok(data, "[^" .. sep .. "]*")
+        local strings = strtok(data, sep)
         local octets = {}
         for k, v in ipairs(strings) do
                 -- exclude empty strings from conversion

src/lua/zencode_data.lua

@@ -100,7 +100,7 @@
  -- return leftmost and rightmost if definition string indicates
  -- a lua table: dictionary, array or schema
  local function expect_table(definition)
-    local toks = strtok(definition, '[^_]+')
+    local toks = strtok(definition, '_')
     local res = { rightmost = toks[#toks] }
     if res.rightmost == 'array'
        or

src/lua/zencode_http.lua

@@ -125,7 +125,7 @@ When("create the url from ''", function(src)
 	if url:sub(1,7)=='http://' then proto = 'http://' end
 	if url:sub(1,8)=='https://' then proto = 'https://' end
 	ZEN.assert(proto, "Invalid http prefix in url: "..obj:str())
-	local toks = strtok(url, '[^/]+') -- second is the host
+	local toks = strtok(url, '/') -- second is the host
 	local res = _is_valid_host(toks[2])
 	ZEN.assert(not res, res)
 	ACK.url = obj

src/lua/zencode_w3c.lua

@@ -27,7 +27,7 @@ local function import_did_document(doc)
     ZEN.assert(doc.id and #doc.id < 256, 'Invalid DID document: id not found')
     ZEN.assert(doc['@context'], 'Invalid DID document: @context not found')
     ZEN.assert(#JSON.encode(doc, 'string') < 4096, 'DID document too large')
-    local did_components = strtok(doc.id, '[^:]*')
+    local did_components = strtok(doc.id, ':')
     -- schme: did
     ZEN.assert(did_components[1] == 'did',
 	       'Invalid DID document: invalid scheme')
@@ -113,7 +113,7 @@ ZEN.add_schema(
 
 -- return { r , s } table suitable for signature verification
 local function jws_octet_to_signature(obj)
-    local toks = strtok(OCTET.to_string(obj), '[^.]*')
+    local toks = strtok(OCTET.to_string(obj), '.')
     -- header parsing may be skipped
     -- local header = JSON.decode( OCTET.from_url64(toks[1]):to_string() )
     local res = {}
@@ -274,7 +274,7 @@ IfWhen(
                     'The object has no signature: ' .. src)
         ZEN.assert(document.proof.verificationMethod,
                     'The proof inside '..src..' has no verificationMethod')
-        local data = strtok(O.to_string(document.proof.verificationMethod), '[^#]*' )
+        local data = strtok(O.to_string(document.proof.verificationMethod), '#' )
         local signer_id = O.from_string(data[1])
         ZEN.assert(signer_id == signer_document.id,
                     'The signer id in proof is different from the one in '..signer_did_doc)
@@ -299,7 +299,7 @@ When(
         ZEN.assert(doc.service, 'service not found')
         ACK.serviceEndpoint = {}
         for _, service in pairs(doc.service) do
-            local name = strtok(O.to_string(service.id), '[^#]*')[2]
+            local name = strtok(O.to_string(service.id), '#')[2]
             ACK.serviceEndpoint[name] = service.serviceEndpoint
         end
         new_codec('serviceEndpoint', { encoding = 'string',
@@ -317,10 +317,10 @@ When(
         ACK.verificationMethod = {}
 
         for _, ver_method in pairs(doc.verificationMethod) do
-            local pub_key_name = strtok(O.to_string(ver_method.id), '[^#]*')[2]
+            local pub_key_name = strtok(O.to_string(ver_method.id), '#')[2]
             if pub_key_name == ETHEREUM_ADDRESS then
                 local address = strtok(
-                    O.to_string(ver_method.blockchainAccountId), '[^:]*' )[3]
+                    O.to_string(ver_method.blockchainAccountId), ':' )[3]
                 ACK.verificationMethod[pub_key_name] = O.from_hex(address)
             else
                 local pub_key = O.to_string(ver_method.publicKeyBase58)
@@ -351,4 +351,4 @@ When(
         ZEN.CODEC[pk_name] = guess_conversion(ACK[pk_name], pk_name)
         ZEN.CODEC[pk_name].name = pk_name
     end
-)
+)

src/lua/zenroom_common.lua

@@ -295,19 +295,6 @@ function help(module)
    -- end
 end
 
--- TODO: optimize in C using strtok
-local function split(src,pat)
-   local tbl = {}
-   src:gsub(pat, function(x) tbl[#tbl+1]=x end)
-   return tbl
-end
-function strtok(src, pat)
-   if not src then return { } end
-   pat = pat or "%S+"
-   ZEN.assert(luatype(src) == "string", "strtok error: argument is not a string")
-   return split(src, pat)
-end
-
 local oldtonumber = tonumber
 function tonumber(obj, ...)
     if type(obj) == "zenroom.float" then

src/zen_parse.c

@@ -172,6 +172,62 @@ static int lua_unserialize_json(lua_State* L) {
 	return 0;
 }
 
+char* strtok_single(char* str, char const* delims)
+{
+    static char* src = NULL;
+    char *p, *ret = NULL;
+
+    if (str != NULL)
+        src = str;
+
+    if (src == NULL)
+        return NULL;
+
+    if ((p = strpbrk(src, delims)) != NULL) {
+        *p = 0;
+        ret = src;
+        src = ++p;
+    } else {
+        ret = src;
+        src = NULL;
+    }
+
+    return ret;
+}
+
+static int lua_strtok(lua_State* L) {
+	const char DEFAULT_SEP[] = " ";
+
+	char copy[MAX_FILE];
+	char *sep = DEFAULT_SEP;
+
+	const char *in;
+	size_t size;
+	register int i = 1;
+	register char *token;
+
+	in = luaL_checklstring(L, 1, &size);
+
+	if (lua_gettop(L) > 1) {
+		sep = luaL_checklstring(L, 2, NULL);
+	}
+
+	lua_newtable(L);
+
+	memcpy(copy, in, size+1);
+
+	token = strtok_single(copy, sep);
+	while(token != NULL) {
+		lua_pushlstring(L, token, strlen(token));
+
+		lua_rawseti(L, -2, i);
+
+		token = strtok_single(NULL, sep);
+		i = i + 1;
+	}
+	return 1;
+}
+
 void zen_add_parse(lua_State *L) {
 	// override print() and io.write()
 	static const struct luaL_Reg custom_parser [] =
@@ -180,6 +236,7 @@ void zen_add_parse(lua_State *L) {
 		  {"trim", lua_trim_spaces},
 		  {"trimq", lua_trim_quotes},
 		  {"jsontok", lua_unserialize_json},
+		  {"strtok", lua_strtok},
 		  {NULL, NULL} };
 	lua_getglobal(L, "_G");
 	luaL_setfuncs(L, custom_parser, 0);  // for Lua versions 5.2 or greater

test/lua/w3c-vc.lua

@@ -52,7 +52,7 @@ local v_proof = cred_table.proof
 v_cred_table.proof = nil
 v_cred_oct = ZEN.serialize(v_cred_table)
 assert(v_cred_oct == cred_oct) -- check serialization is deterministic
-local v_jws = strtok(v_proof.jws,'[^.]*')
+local v_jws = strtok(v_proof.jws,'.')
 -- parse header
 local v_header = JSON.decode( v_jws[1] )
 I.print(v_header)

test/vectors/check_eddsa.lua

@@ -6,7 +6,7 @@ local function newline_iter(text)
 end
 
 for line in newline_iter(DATA) do
-   local tokens = strtok(line, "[^:]*")
+   local tokens = strtok(line, ":")
    local sk = O.from_hex(tokens[1]:sub(1, 64))
    local pk = O.from_hex(tokens[2])
    assert(ED.pubgen(sk) == pk)